Add: Incremental Books

For new exchanges, TAIFEX and CBOE

.config/dotnet-tools.json

@@ -9,7 +9,7 @@
       ]
     },
     "fable": {
-      "version": "4.24.0",
+      "version": "4.0.6",
       "commands": [
         "fable"
       ]
@@ -25,12 +25,6 @@
       "commands": [
         "fake"
       ]
-    },
-    "dotnet-t4": {
-      "version": "3.0.0",
-      "commands": [
-        "t4"
-      ]
     }
   }
-}
+}

.devcontainer/Dockerfile

@@ -55,19 +55,17 @@ RUN \
    clangd-$clang_major
 
 # CMake
-RUN set -eux \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
-  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
-    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
-  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-    > /etc/apt/sources.list.d/kitware.list \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends cmake \
-  && rm -rf /var/lib/apt/lists/*
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
 
 # Dotnet
-RUN apt-get update && apt-get install -y dotnet8
+RUN apt-get update && apt-get install -y dotnet7
 
 # Node
 ARG node_major=18
@@ -82,14 +80,9 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
-    && apt-get install -y /tmp/emumba-deribit-connector.deb \
-    && rm -f /tmp/emumba-deribit-connector.deb
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
 
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -97,24 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
-# Rakurai
-ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
-RUN --mount=type=secret,id=bitbucket_access_token \
-  set -eux \
-    && apt-get update \
-    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
-    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
-    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
-    # spdlog headers.
-    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
-    && rm -f /tmp/rakuirai.deb
-
-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -1,16 +1,15 @@
 {
-  "build": {
-    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
-    "options": [
-      "--secret",
-      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
-    ]
-  },
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "8gb",
-    "storage": "10gb"
+    "memory": "4gb",
+    "storage": "6gb"
   },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -18,21 +17,34 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "8gb"
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345"
   ],
   "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
   ],
   "customizations": {
     "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
-        "ms-python.python",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools",
-        "xaver.clang-format"
+        "ms-vscode.cpptools"
       ]
     }
   }
-}
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.editorconfig

@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower
 
-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion
 
-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion

.env

@@ -1,64 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Host/Runtime ###
-TZ=Etc/UTC
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
-# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
-# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
-SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
-
-### PubSubServer ###
-PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
-PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-
-### MdPublisher ###
-MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
-MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -11,7 +11,6 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
-!.vscode/*.env
 .vs
 obj/
 *.csproj.user
@@ -55,12 +54,8 @@ trace_data/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/
-/src/**/*appsettings.Local.json
-/src/**/*/Configurations/Local/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local
-.secrets

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git

.vscode/launch.json

@@ -1,62 +1,11 @@
 {
   "version": "0.2.0",
   "configurations": [
-    {
-      "name": "Python Debugger: Current File with Arguments",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "args": "${command:pickArgs}"
-    },
     {
       "name": ".NET Core Attach",
       "type": "coreclr",
       "request": "attach"
     },
-    {
-      "name": "Api Server",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Api Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "spark_api_server.exe",
-      },
-      "linux": {
-        "program": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -69,28 +18,11 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -99,39 +31,7 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "run"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests2",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader2]"
-      ],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
-    {
-      "name": "spreader tests",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": [
-        "test",
-        "[spreader]"
-      ],
+      "args": ["run"],
       "stopAtEntry": false,
       "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
       "environment": [],
@@ -142,30 +42,17 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
         "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "DataStreamQueryAnalyzer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
-          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
         },
       },
       "stopAtEntry": false,
@@ -183,65 +70,11 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "preLaunchTask": "Build Spark Sdk (Debug)",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sdk Demo (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "linux": {
-        "program": "Bts.Spark.Sdk.Demo.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:sdkDemoEnvironment}",
-      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -253,25 +86,7 @@
       "windows": {
         "preLaunchTask": "Build Spark (Debug)",
         "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net8.0-windows",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
-        }
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
@@ -285,136 +100,32 @@
       "windows": {
         "program": "Spark.exe",
         "args": [
-          "--Spark:ConfigServer",
+          "--sparkd",
           "${input:sparkConfigServer}"
         ],
-        "cwd": "build/Debug/bin/net8.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark Test (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "program": "Spark.Console.exe",
-        "cwd": "build/Debug/bin/net8.0",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ]
+        "cwd": "build/Debug/bin/net7.0-windows",
       },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
     {
       "name": "Sparkd",
-      "type": "cppdbg",
+      "type": "coreclr",
       "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
       "program": "",
       "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-        "preLaunchTask": "Build Sparkd (Debug | Clang)",
+        "program": "sparkd",
       },
       "args": [
         "run"
       ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "build/sparkd-debug-clang/server",
       "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
-    {
-      "name": "Sparkd Attach",
-      "type": "cppdbg",
-      "request": "attach",
-      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
+      "console": "integratedTerminal"
+    }
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "dataStreamPublisherConfig",
       "type": "pickString",
@@ -428,46 +139,14 @@
       "default": "MarketDataPublisher",
     },
     {
-      "id": "dataStreamQueryAnalyzerEnvironment",
+      "id": "pubSubServerConfig",
       "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
+      "description": "PubSub Configuration",
       "options": [
-        "Development",
-        "Local",
+        "PubSubServer",
+        "SimpleEchoClient",
       ],
-      "default": "Development",
-    },
-    {
-      "id": "dataStreamPublisherEnvironment",
-      "type": "pickString",
-      "description": "DataStreamPublisher Configuration",
-      "options": [
-        "Local",
-        "ReplayMarketData",
-      ],
-      "default": "Local",
-    },
-    {
-      "id": "pubSubServerEnvironment",
-      "type": "pickString",
-      "description": "PubSubServer Configuration",
-      "options": [
-        "Development",
-        "EchoServer",
-        "Local",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sdkDemoEnvironment",
-      "type": "pickString",
-      "description": "Sdk Demo Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
+      "default": "PubSubServer",
     },
     {
       "id": "sparkConfigServer",
@@ -475,25 +154,5 @@
       "description": "Config Server URL",
       "default": "http://localhost:8000/config",
     },
-    {
-      "id": "sparkEnvironment",
-      "type": "pickString",
-      "description": "Spark Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
   ]
-}
+}

.vscode/python.env

@@ -1 +0,0 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py

.vscode/tasks.json

@@ -172,66 +172,6 @@
         "color": "terminal.ansiCyan"
       },
     },
-    {
-      "label": "Build Spark Sdk",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark Sdk (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark_sdk.cmd",
-      },
-      "linux": {
-        "command": "tools/build_spark_sdk.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
     {
       "label": "Build Sparkd",
       "group": "build",
@@ -411,129 +351,6 @@
         "id": "wrench"
       },
     },
-    {
-      "label": "Run Api Server",
-      "dependsOn": "Build Api Server",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug)",
-      "dependsOn": "Build Api Server (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release)",
-      "dependsOn": "Build Api Server (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Api Server (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "spark_api_server.exe",
-      },
-      "linux": {
-        "command": "spark_api_server",
-      },
-      "args": [
-        "run",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
-        }
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
     {
       "label": "Run DataStreamQueryAnalyzer",
       "dependsOn": "Build DataStream",
@@ -541,11 +358,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -560,11 +378,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -579,11 +398,12 @@
       "command": "",
       "windows": {
         "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
-          "env": {
-            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
-          },
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -602,11 +422,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -624,11 +445,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -646,11 +468,12 @@
       "windows": {
         "command": "DataStreamPublisher.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -668,11 +491,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -690,11 +514,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -711,11 +536,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -733,11 +559,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -754,11 +581,12 @@
       "windows": {
         "command": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
       "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
-        "env": {
-          "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-        },
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -773,7 +601,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -789,7 +617,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -805,22 +633,7 @@
       "windows": {
         "command": "Spark.exe",
         "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Skip Build)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
         },
       },
       "icon": {
@@ -845,7 +658,7 @@
         "${input:sparkConfigQuery}",
       ],
       "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
       },
       "icon": {
         "id": "run",
@@ -871,24 +684,6 @@
         "color": "terminal.ansiGreen",
       },
     },
-    {
-      "label": "Run Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
     {
       "label": "Run Sparkd Config Query",
       "dependsOn": "Build Sparkd",
@@ -910,65 +705,6 @@
         "color": "terminal.ansiGreen",
       },
     },
-    {
-      "label": "Test Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-      "command": "",
-      "windows": {
-        "command": "Spark.Console.exe",
-        "args": [
-          "test",
-          "${input:sparkTest}"
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
-        }
-      },
-    },
-    {
-      "label": "Test Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
-    {
-      "label": "Test Sparkd (Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "test",
-        "${input:sparkdTest}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "coverage",
-        "color": "terminal.ansiYellow",
-      },
-    },
     {
       "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
       "type": "process",
@@ -1024,17 +760,6 @@
     },
   ],
   "inputs": [
-    {
-      "id": "apiServerEnvironment",
-      "type": "pickString",
-      "description": "Api Server Configuration",
-      "options": [
-        "Development",
-        "Local",
-        "Staging",
-      ],
-      "default": "Development",
-    },
     {
       "id": "buildConfiguration",
       "type": "pickString",
@@ -1045,16 +770,6 @@
       ],
       "default": "Debug",
     },
-    {
-      "id": "dataStreamQueryAnalyzerEnvironment",
-      "type": "pickString",
-      "description": "DataStreamQueryAnalyzer Configuration",
-      "options": [
-        "Development",
-        "Local",
-      ],
-      "default": "Development",
-    },
     {
       "id": "sparkdCmakePreset",
       "type": "pickString",
@@ -1078,30 +793,26 @@
       "default": "sparkd-debug-clang",
     },
     {
-      "id": "dataStreamPublisherEnvironment",
+      "id": "dataStreamPublisherConfig",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "Development",
-        "MarketData",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "Development",
-        "EchoServer",
-        "Production",
-        "ProductionAws",
-        "Staging",
+        "PubSubServer",
+        "SimpleEchoClient",
       ],
-      "default": "Development",
+      "default": "PubSubServer",
     },
     {
       "id": "sparkConfigQuery",
@@ -1112,16 +823,6 @@
       "id": "sparkdConfigQuery",
       "type": "promptString",
       "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
-    {
-      "id": "sparkdTest",
-      "type": "promptString",
-      "description": "Test to run",
-    },
+    }
   ]
 }

Bts.Spark.Sdk.sln

@@ -1,102 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|Any CPU.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x64.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.ActiveCfg = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Debug|x86.Build.0 = Debug|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|Any CPU.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x64.Build.0 = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.ActiveCfg = Release|x64
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C}.Release|x86.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|Any CPU.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x64.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.ActiveCfg = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Debug|x86.Build.0 = Debug|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|Any CPU.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x64.Build.0 = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.ActiveCfg = Release|x64
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9}.Release|x86.Build.0 = Release|x64
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x64.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Debug|x86.Build.0 = Debug|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x64.Build.0 = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.ActiveCfg = Release|Any CPU
-		{37E4E890-D025-4729-90F9-6F95862F47BC}.Release|x86.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Debug|x86.Build.0 = Debug|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x64.Build.0 = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}.Release|x86.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x64.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Debug|x86.Build.0 = Debug|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-	EndGlobalSection
-EndGlobal

CMakeLists.txt

@@ -1,10 +1,7 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
 set(CMAKE_CXX_STANDARD 20)
 
-get_version_from_git()
 if(WIN32)
   set(platform "windows")
 else()
@@ -21,10 +18,10 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
-set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
-set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
-set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
+set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
 set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
@@ -36,18 +33,25 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
-include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
-link_directories(${EXTERNALS_LIB})
+include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${SPARK_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-if(DEFINED APP_DATA_DIR)
-  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
   # subdirs
-  add_subdirectory(src/lib_spark)
+  add_subdirectory(src/lib)
 else()
+  # make sparkd look for libspark.so in its directory
+  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  set(CMAKE_INSTALL_RPATH ".")
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
+
   if(BUILD_SPARKD)
     # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
     # read by their code). We override some below.
@@ -85,15 +89,9 @@ else()
     add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
 
     # ** Binance USDS Futures **
-    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
-    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
-    add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
-
-    # ** Binance Coin Futures **
-    add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-    add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-    add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
 
     # ** Deribit **
     add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -112,15 +110,11 @@ else()
   endif()
 
   # subdirs
-  add_subdirectory(src/lib_spark)
   if (BUILD_SPARKD)
-    add_subdirectory(src/lib_python_interop)
-    add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance)
-    add_subdirectory(src/lib_okx_execution)
-    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/server/sparkd)
     add_subdirectory(src/data_files)
   endif()
+  add_subdirectory(src/lib)
   if (BUILD_ADMIN_WEB_APP)
     add_subdirectory(src/Spark.Admin.WebApp)
   endif()
@@ -130,4 +124,34 @@ else()
   if(BUILD_SDK)
     add_subdirectory(src/sdk)
   endif()
+
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
+
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+  endif()
 endif()

CMakePresets.json

@@ -8,41 +8,36 @@
 
   "configurePresets": [
     {
-      "name": "spark-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
-      }
-    },
-
-    {
-      "name": "spark-windows",
-      "inherits": "spark-base",
-      "displayName": "Spark",
+      "name": "spark",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
       "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build/${presetName}",
       "environment": {
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
+        "CMAKE_CXX_STANDARD": "20",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
-        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
       }
     },
 
@@ -58,20 +53,16 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_CXX_STANDARD": "20",
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
-        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
-        "DATABENTO_USE_EXTERNAL_JSON": {
           "type": "BOOL",
           "value": "ON"
         },
+        "CMAKE_CXX_STANDARD": "20",
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
         "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
         "BUILD_SDK": {
           "type": "BOOL",
@@ -80,7 +71,8 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
       }
     },
 
@@ -95,7 +87,7 @@
           "value": "ON"
         },
         "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
       }
     },
 
@@ -106,7 +98,7 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
       }
     },
 
@@ -159,17 +151,22 @@
     },
 
     {
-      "name": "spark-linux",
-      "inherits": "spark-base",
-      "displayName": "Spark",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
+      "name": "spark-api-server-windows",
+      "inherits": "spark",
+      "displayName": "Spark Api Server (Windows)",
+      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
+      }
+    },
+
+    {
+      "name": "spark-api-server-linux",
+      "inherits": "sparkd",
+      "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "platform": "linux",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -178,26 +175,17 @@
           "type": "BOOL",
           "value": "OFF"
         },
-        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
         "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
-        "DATABENTO_USE_EXTERNAL_JSON": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "EXTERNALS_ROOT": "${sourceDir}/externals/linux"
+        }
       }
     },
 
@@ -217,32 +205,93 @@
       "cacheVariables": {
         "CCAPI_LOG_LEVEL": "Trace"
       }
-    }
-  ],
+    },
 
-  "buildPresets": [
     {
-      "name": "spark-windows",
+      "name": "data-stream-suite-windows",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "hidden": true,
-      "configurePreset": "spark-windows"
+      "inherits": "spark",
+      "displayName": "Data Stream Suite (Windows)",
+      "description": "Data Stream utils ported over from Edge",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
     },
 
     {
-      "name": "spark-windows-debug",
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true
+    },
+
+    {
+      "name": "spark-debug",
       "displayName": "Spark (Debug)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-windows-release",
+      "name": "spark-release",
       "displayName": "Spark (Release)",
-      "inherits": "spark-windows",
+      "inherits": "spark",
+      "configurePreset": "spark",
       "configuration": "Release"
     },
 
@@ -253,7 +302,16 @@
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
-      "hidden": true
+      "hidden": true,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
     },
 
     {
@@ -285,27 +343,52 @@
     },
 
     {
-      "name": "spark-linux",
+      "name": "spark-api-server-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true,
+      "configurePreset": "spark-api-server-windows"
+    },
+
+    {
+      "name": "spark-api-server-windows-debug",
+      "displayName": "Spark Api Server (Windows | Debug)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-api-server-windows-release",
+      "displayName": "Spark Api Server (Windows | Release)",
+      "inherits": "spark-api-server-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "spark-api-server-linux",
       "condition": {
         "type": "equals",
         "lhs": "${hostSystemName}",
         "rhs": "Linux"
       },
       "hidden": true,
-      "configurePreset": "spark-linux"
+      "configurePreset": "spark-api-server-linux"
     },
 
     {
-      "name": "spark-linux-debug",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Debug)",
+      "name": "spark-api-server-linux-debug",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Debug)",
       "configuration": "Debug"
     },
 
     {
-      "name": "spark-linux-release",
-      "inherits": "spark-linux",
-      "displayName": "Spark (Linux | Release)",
+      "name": "spark-api-server-linux-release",
+      "inherits": "spark-api-server-linux",
+      "displayName": "Spark Api Server (Linux | Release)",
       "configuration": "Release"
     },
 
@@ -322,6 +405,34 @@
       "configurePreset": "sparkd-ccapi-diag",
       "displayName": "Debug (ccapi diag)",
       "inherits": "sparkd"
+    },
+
+    {
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Release"
     }
   ]
 }

Spark.Api.Server.sln

@@ -1,11 +1,10 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.3.32922.545
 MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
@@ -15,584 +14,66 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x86 = Debug|x86
 		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
 		Release|x64 = Release|x64
-		Release|Any CPU = Release|Any CPU
-		Release|x86 = Release|x86
 		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x86.Build.0 = Debug|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.ActiveCfg = Release|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x86.Build.0 = Release|Any CPU
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x86.Build.0 = Debug|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.ActiveCfg = Release|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x86.Build.0 = Release|Any CPU
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x86.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x86.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x86.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x86.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x86.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x86.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x86.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x86.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x86.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x86.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x86.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x86.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x86.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x86.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x86.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x86.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x86.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x86.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x86.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x86.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x86.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x86.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x86.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|Any CPU.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x86.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Debug|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x64.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.ActiveCfg = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.Release|x86.Build.0 = Release|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
 	EndGlobalSection

cmake/ExportToParent.cmake

@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()

cmake/GitVersion.cmake

@@ -1,48 +0,0 @@
-include_guard(GLOBAL)
-
-function(get_version_from_git)
-  find_package(Git QUIET)
-  if(NOT Git_FOUND)
-    message(WARNING "Git not found")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} describe --tags --always
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_TAG
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    RESULT_VARIABLE GIT_RESULT
-  )
-
-  if(NOT GIT_RESULT EQUAL 0)
-    message(WARNING "Failed to get git tag")
-    return()
-  endif()
-
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
-  set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
-
-  string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
-  if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
-    set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
-    set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
-    set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
-
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 
-    set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
-    set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
-  else()
-    message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
-  endif()
-endfunction()

cmake/InitLibrary.cmake

@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES AND NOT WIN32)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()

cmake/InitPythonModule.cmake

@@ -1,71 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(init_python_module)
-  set(_options)
-  set(_one_value_args
-    NAME
-  )
-  set(_multi_value_args
-    SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
-
-  set(module_proj ${_name}_module)
-  set(${module_proj}_SOURCES ${_sources})
-  pybind11_add_module(${module_proj} SHARED ${_sources})
-  target_link_libraries(
-    ${module_proj}
-    PUBLIC
-      pybind11::embed
-      spark
-      ${_name}
-  )
-  target_include_directories(
-    ${module_proj}
-    PRIVATE
-      $<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
-  )
-  target_include_directories(
-    ${module_proj}
-    PUBLIC
-      ${Python3_INCLUDE_DIRS}
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-    DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
-    COMMENT "Generating py stubs"
-  )
-  add_custom_target(
-    "${_name}_py_stubs"
-    ALL
-    DEPENDS
-      ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
-  )
-  install(
-    TARGETS ${module_proj}
-    EXPORT ${module_proj}Targets
-    RUNTIME
-      DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE
-      DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  )
-  foreach(dir ${pybind11_INCLUDE_DIRS})
-    install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-  endforeach()
-  install(
-    EXPORT ${module_proj}Targets
-    FILE ${module_proj}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-endfunction()

cmake/InitPythonPlugin.cmake

@@ -1,65 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-
-function(init_python_plugin)
-  set(_options)
-  set(_one_value_args NAME PLUGIN_NAME)
-  set(_multi_value_args SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  if (NOT DEFINED _ARGS_PLUGIN_NAME)
-    set(_plugin_name ${_name}_plugins)
-  endif()
-  set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
-  if (NOT DEFINED _ARGS_SOURCES)
-    file(
-      GLOB_RECURSE ${_plugin_name}_SOURCES
-      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-      ${_plugin_name}/*.py)
-  endif()
-  set(src_outs)
-  foreach(_src ${${_plugin_name}_SOURCES})
-    set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
-    set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
-    add_custom_command(
-      OUTPUT ${_src_out}
-      COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
-      DEPENDS ${_src_in}
-      COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
-    )
-    list(APPEND src_outs ${_src_out})
-  endforeach()
-  
-  add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
-  
-  add_library(${_plugin_name} INTERFACE)
-  add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-      INTERFACE_PYTHON_SCRIPTS_DIR
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
-  )
-  
-  set_target_properties(
-    ${_plugin_name}
-    PROPERTIES
-    INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
-  )
-  
-  install(
-    TARGETS ${_plugin_name}
-    EXPORT ${_plugin_name}Targets
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
-  
-  install(
-    EXPORT ${_plugin_name}Targets
-    FILE ${_plugin_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-  install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
-endfunction()

cmake/InstallBasic.cmake

@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()

cmake/PackBasic.cmake

@@ -1,44 +0,0 @@
-include_guard(DIRECTORY)
-include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
-  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    VERSION ${PROJECT_VERSION}
-    COMPATIBILITY AnyNewerVersion
-  )
-
-  configure_package_config_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
-  )
-
-  install(
-    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-endfunction()

cmake/Protobuf.cmake

@@ -0,0 +1,145 @@
+set(Protobuf_PROTOC_EXECUTABLE "${SPARK_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${SPARK_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+
+function(Protobuf_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
+        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${SRCS}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp SRCS HDRS)
+  set(${SRCS})
+  set(${HDRS})
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    if(WIN32)
+      add_custom_command(
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
+          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc"
+               "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES}
+                ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${SRCS}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${SRCS}
+      ${${SRCS}}
+      PARENT_SCOPE)
+  set(${HDRS}
+      ${${HDRS}}
+      PARENT_SCOPE)
+
+endfunction()
+

cmake/ProtobufGenerateCpp.cmake

@@ -1,83 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    if (WIN32)
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc.exe")
-    else()
-      set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-    endif()
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    # TODO: Pass proto sources by relative path (so this fn can be called from
-    # arbitrary subdirs).
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_abs_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
-

cmake/ProtobufGenerateCppConfig.cmake

@@ -1,61 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
-

cmake/ProtobufGenerateFs.cmake

@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()

cmake/ProtobufGenerateJs.cmake

@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()

cmake/SetFirstDefined.cmake

@@ -1,10 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
-      break()
-    endif()
-  endforeach()
-endmacro()

cmake/SymLink.cmake

@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
-  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
-  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
-  )
-endfunction()
-
-function(sym_link_file)
-  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
-  )
-  if (DEFINED _ARGS_INSTALL_DEST)
-    install(
-      FILES
-      ${_ARGS_SOURCE}
-      DESTINATION ${_ARGS_INSTALL_DEST}
-    )
-  endif()
-endfunction()
-
-function(sym_link_local_lib LIB_NAME)
-  set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
-  file(
-    CREATE_LINK
-      "${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
-    SYMBOLIC
-  )
-endfunction()

cmake/SymLinkPythonModule.cmake

@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-
-function(sym_link_python_module)
-  set(_options)
-  set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
-                        "${_multi_value_args}" ${ARGN})
-  if(NOT DEFINED _ARGS_LIB_NAME)
-    message(
-      FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
-  endif()
-  set(_lib_name ${_ARGS_LIB_NAME})
-  set(_module_name ${_ARGS_MODULE_NAME})
-  if(NOT DEFINED _ARGS_MODULE_NAME)
-    set(_module_name ${_lib_name}_module)
-  endif()
-  set(_plugin_name ${_ARGS_PLUGIN_NAME})
-  set(_deps ${_lib_name} ${_module_name})
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
-  endif()
-  add_dependencies(${PROJECT_NAME} ${_deps})
-  sym_link_local_lib(${_lib_name})
-  set(_module_suffix cpython-310-x86_64-linux-gnu.so)
-  sym_link_file(
-    SOURCE
-    ${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
-    DEST
-    ${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
-    INSTALL_DEST
-    ${CMAKE_INSTALL_BINDIR}/)
-  if(DEFINED _ARGS_PLUGIN_NAME)
-    get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
-    sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
-                 ${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
-  endif()
-endfunction()

cmake/TernaryOperator.cmake

@@ -1,9 +0,0 @@
-include_guard(GLOBAL)
-
-macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
-  if(${CONDITION})
-    set(${RESULT} ${TRUE_VALUE})
-  else()
-    set(${RESULT} ${FALSE_VALUE})
-  endif()
-endmacro()

cmake/config_proto_files.h.in

@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs

cmake/version.h.in

@@ -1,11 +0,0 @@
-#ifndef BTS_SPARK_VERSION_H
-#define BTS_SPARK_VERSION_H
-
-#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
-#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
-#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
-#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
-#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
-#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
-
-#endif

cmake/vs_filters.cmake

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Explicitly set policy to convert relative paths to absolute (although this is a default policy,
+# setting it explicitly prevents a compile-warning)
+cmake_policy(SET CMP0076 NEW)
+
+# add protos to the library to facilitate source_group
+target_sources(libspark PUBLIC ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})
+
+function(assign_source_group)
+  foreach(_source IN ITEMS ${ARGN})
+    if(IS_ABSOLUTE "${_source}")
+      file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+    else()
+      set(_source_rel "${_source}")
+    endif()
+    get_filename_component(_source_path "${_source_rel}" PATH)
+    string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+    source_group("${_source_path_msvc}" FILES "${_source}")
+    set(SOURCE_FILES "${_source}")
+  endforeach()
+endfunction(assign_source_group)
+
+assign_source_group(${SparkLib_Sources} ${Config_Protos} ${NonConfig_Protos} ${Service_Protos})

compose.yaml

@@ -1,254 +0,0 @@
-services:
-  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-shell:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-    build:
-      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: "do sleep 1; done"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-symbols:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-  sparkd-multicast:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
-  
-  sparkd-multicast-cert:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet8
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet8
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
-
-  pubsub:
-    volumes:
-      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
-    build:
-      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
-      args:
-        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
-        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
-    ports:
-      - 22019:22019
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
-        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
-          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/pubsub/conf/.env.local
-          set +a
-        fi
-        ./PubSubServer"
-    env_file:
-      - .env
-      - ${PUBSUB__CONFIG_DIR}/.env
-
-  md-publisher:
-    volumes:
-      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
-      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
-    build:
-      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
-      args:
-        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
-        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
-        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
-          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env
-          set +a
-        fi
-        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
-          set +a
-        fi
-      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
-    env_file:
-      - .env
-      - ${MD_PUBLISHER__CONFIG_DIR}/.env
-
-  api-server:
-    depends_on:
-      - sparkd
-    ports:
-      - 8008:8008
-      - 9443:9443
-    build:
-      dockerfile: src/client/spark_api_server/Dockerfile
-    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
-    environment:
-      - ASPNETCORE_ENVIRONMENT=Staging
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
-  pubsub_data:
-    name: "pubsub_22019"
-  md_publisher_data:
-    name: "md_publisher_22019"
-  historic_md:
-    name: "historic_md"
-  md_recording:
-    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
-    external: true

doc/api/content/client/getting-started/services.md

@@ -124,8 +124,8 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/OrderService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
 using namespace bts::api;
 ServiceContext ctx;
 auto orders = ctx.locate<orderman::OrderService>();

doc/api/content/client/services/marketdata.md

@@ -29,9 +29,9 @@ namespace sample {
 ```
 
 ```cpp
-#include "spark/api/public/ServiceContext.h"
-#include "spark/domain_model/Security.h"
-#include "spark/api/services/MarketDataService.client.pb.h"
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 #include <thread>
 
 using namespace bts::api;

doc/sparkd-security-id-lifecycle.md

@@ -1,942 +0,0 @@
-# Security ID handling in `sparkd`
-
-This document traces how security identifiers are created, translated, persisted,
-indexed, and published by the `sparkd` market-data paths. It focuses on the
-legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
-
-## Executive summary
-
-`sparkd` uses two identifiers that must not be confused:
-
-| Identifier | Meaning | Typical field | Lifetime |
-|---|---|---|---|
-| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
-| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
-
-`invalid_security_key` is `0`; a security with key `0` is not yet publicly
-discoverable through the shared-memory consumer path.
-
-The two principal paths differ:
-
-- **MDP3/OnixS:** the definition callback sets only the CME ID in
-  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
-  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
-  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
-  calls `MDProducerHandle::set_key()`.
-- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
-  currently uses the ID already present on the incoming domain security
-  (`use_given_sec_key == true`). Live adapters normally obtain that ID
-  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
-  it to both `FixedSecurity::key` and `numeric_id`, so
-  `MDProducer::add_security()` publishes it synchronously. Only the listener's
-  own fallback `IDGenerator` call is bypassed by the current constant.
-
-## End-to-end diagrams
-
-### CME MDP3 / OnixS
-
-```mermaid
-sequenceDiagram
-    participant FE as OnixS feed-engine thread
-    participant L as MDP3Listener
-    participant P as MDProducer / shared memory
-    participant Q as SecurityIDRecorder queues
-    participant D as sparkd dispatcher thread
-    participant DB as key_ids.sqlite
-
-    FE->>L: instrument definition (CME SecurityID)
-    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
-    L->>P: add_security(FixedSecurity)
-    P->>P: allocate hot + cold entries with key 0
-    P->>Q: ensure_key(handle)
-    L->>L: index[CME SecurityID] = handle
-    Note over FE,L: Books can route by CME ID while the Spark key is pending
-    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
-    Q->>DB: find (namespace, descriptor), or insert a new row
-    DB-->>Q: SQLite rowid = Spark security key
-    Q->>P: handle.set_key(rowid)
-    P->>P: hot.key first, then cold security.key, then increment security_count
-    Note over P: The security is now public to shared-memory consumers
-```
-
-### Generalized/newer pipeline (current behavior)
-
-```mermaid
-sequenceDiagram
-    participant A as Venue adapter / plugin
-    participant DB as IDGenerator / key_ids.sqlite
-    participant G as GeneralMarketDataListener
-    participant I as GeneralSecurityIndex
-    participant P as MDProducer / shared memory
-
-    A->>DB: get_or_add(venue, type, adapter key)
-    DB-->>A: SQLite rowid as domain SecurityID
-    A->>G: on_securities(domain Security list with ID)
-    G->>G: _find_sec() - use incoming security_common.id
-    G->>G: _copy_security_common() - key = id, numeric_id = id
-    G->>P: add_security(FixedSecurity with non-zero key)
-    P->>P: set_key() during add_security()
-    G->>I: index[id] = handle
-    Note over G,P: No asynchronous ID assignment in the current configuration
-```
-
-## Data-flow charts
-
-The sequence diagrams above emphasize timing and thread transitions. These
-flowcharts emphasize which identifier is carried through each representation
-and where data is retained.
-
-### MDP3 / legacy data flow
-
-```mermaid
-flowchart TD
-    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
-
-    ONIXS --> DEF["MDP3Listener definition callback"]
-    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
-
-    FIXED --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
-    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
-
-    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
-
-    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
-    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
-    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
-    UNRESOLVED --> CMEINDEX
-    CMEINDEX --> LEGS
-    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
-
-    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
-    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
-    INSERT --> ROWID
-
-    ROWID --> SETKEY["MDProducerHandle::set_key"]
-    SETKEY --> HOT["Write hot-entry key"]
-    HOT --> COLD["Write cold security key"]
-    COLD --> COUNT["Increment security_count"]
-    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
-
-    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
-    UPDATE --> CMEINDEX
-    CMEINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> LIVE["Update live shared-memory data"]
-    PUBLIC --> LIVE
-```
-
-The CME index is usable before the Spark key is assigned, so feed updates can
-reach the internal handle while the security is still hidden from consumers.
-The persistent rowid crosses the publication boundary only at `set_key()`.
-
-### Generalized/newer data flow
-
-```mermaid
-flowchart TD
-    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
-
-    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
-    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
-    IDINPUT --> MUTEX["IDGenerator global mutex"]
-    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid"]
-    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
-    INSERT --> ROWID
-    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
-    IDTYPE -- Recorded or pre-keyed --> DOMAINID
-
-    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
-    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
-    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
-    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
-
-    CONVERT --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
-    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
-    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
-
-    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
-    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
-    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
-    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
-    PYHOT --> EVENTID
-    PYHOT -. uses same mutex and SQLite path .-> MUTEX
-    EVENTID --> CCMAP
-    CCMAP --> GENERALINDEX
-    GENERALINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
-    SHM --> SHMUPDATE
-
-```
-
-In the normal newer live path, persistence occurs before the definition enters
-the generalized listener. The DB call therefore adds synchronous adapter-side
-latency, but publication needs no later timer. Most adapters use an in-memory
-map for subsequent events; the dotted Python path is the documented exception.
-
-## The publication boundary
-
-`MDProducerHandle::set_key()` is the definitive publication point for the
-legacy shared-memory representation. Its write ordering is intentional:
-
-1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
-2. Write the key into the cold security entry.
-3. Increment the cold header's public `security_count`.
-
-The cold security key is written last because consumers treat a non-zero cold
-key as visibility. At that moment the corresponding hot entry must already be
-addressable. `set_key()` only permits the transition from `0` to one key; it is
-not a key-update mechanism.
-
-## Identifier representations
-
-The names are historically inconsistent. In particular, many protobuf fields
-named `security_id` contain the **Spark key**, not the exchange's ID.
-
-| Layer | Exchange/feed identifier | Spark identifier |
-|---|---|---|
-| MDP3 wire/OnixS | `securityId()` | not present |
-| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
-| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
-| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
-| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
-| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
-| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
-| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
-
-The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
-shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
-64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
-allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
-no explicit checked conversion.
-
-## Which parts use exchange IDs, descriptors, or Spark security IDs?
-
-The three identifier classes serve different jobs:
-
-- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
-- **Descriptor/natural key:** recognizes a definition logically at the
-  persistence, discovery, or human-facing boundary.
-- **Spark security ID/key:** joins the instrument to books, orders, fills,
-  positions, and services after publication.
-
-The core downstream system does not repeatedly identify a security from its
-description. The description is primarily used to obtain a Spark key; after
-that boundary, most joins use the numeric Spark key.
-
-### Component-level usage matrix
-
-| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
-|---|---|---|---|
-| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
-| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
-| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
-| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
-| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
-| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
-| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
-| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
-| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
-| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
-| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
-| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
-| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
-| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
-| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
-| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
-| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
-| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
-| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
-| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
-
-### Exchange/external IDs: where they stop
-
-In MDP3 the separation is explicit:
-
-```text
-CME SecurityID
-    -> FixedSecurity.numeric_id
-    -> CMESecurityIndex lookup
-    -> MDProducerHandle
-    -> Spark security_key for downstream publication
-```
-
-The external CME ID continues to route feed messages but does not become the
-order/position key. This permits the same downstream Spark key to survive a
-restart as long as the logical descriptor mapping remains in
-`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
-
-The generalized path collapses more of these roles:
-
-```text
-exchange/vendor identity
-    -> adapter calls IDGenerator
-    -> domain::SecurityID
-    -> FixedSecurity.key and FixedSecurity.numeric_id
-    -> GeneralSecurityIndex and downstream Spark key
-```
-
-`SecurityCommon::exchange_sec_id` may still contain the original textual
-external identifier, but `GeneralMarketDataListener` does not use it to route
-events and does not put it in `numeric_id`. The adapter must translate later
-events to `domain::SecurityID` before invoking the listener.
-
-### Descriptor/name users
-
-Descriptors are used in three main places:
-
-1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
-   descriptor, while `IDGenerator` uses its caller-provided `name`.
-2. **Definition lookup and discovery:** `SecurityCache` supports
-   venue/type/name lookup; symbol filters and administrative tools use security
-   metadata to select or display instruments.
-3. **Diagnostics:** logs include symbols and descriptions so humans can
-   understand a numeric key.
-
-Descriptions generally do not drive the high-rate book, trade, order, fill, or
-position paths. Those paths use numeric IDs after definition resolution.
-
-It is important that “descriptor” does not mean one universal canonical field:
-
-- Legacy CME uses Globex `security_desc` from tag 55.
-- Many generalized adapters use `sec.desc()` or an exchange instrument name.
-- Databento uses its numeric instrument ID converted to a string.
-- Python uses `exchange_security_id`.
-
-Therefore, two adapters may describe the same economic contract but request
-different logical rows and receive different Spark IDs.
-
-### Spark security-ID users
-
-Once a security is published, its Spark key is the dominant system identifier:
-
-```mermaid
-flowchart LR
-    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
-    KEY --> MDC["MDConsumer and book subscriptions"]
-    KEY --> MBO["MBO snapshots and queries"]
-    KEY --> OMS["Orders, algorithms, and spread legs"]
-    KEY --> FILLS["Fills and execution records"]
-    KEY --> POS["Positions, PNL, and risk"]
-    KEY --> API["Services and client APIs"]
-    KEY --> REC["Relay and market-data recordings"]
-    KEY --> SECDB["Serialized security definitions"]
-```
-
-Fields named `security_id`, `security_key`, or top-level `Security.id()` in
-these downstream areas almost always mean the Spark key. They should not be
-compared directly with CME `SecurityID`, Databento `instrument_id`, or another
-venue/vendor identifier without going through the relevant adapter/index
-mapping.
-
-## Persistence and allocation
-
-### The authoritative key mapping: `key_ids.sqlite`
-
-Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
-file is:
-
-```text
-<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
-```
-
-The database contains one table and two indexes:
-
-```sql
-CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
-CREATE UNIQUE INDEX key_index ON store(ns, key);
-CREATE INDEX ns_index ON store(ns);
-```
-
-The SQLite `rowid` is the generated Spark key. `ns` is normally
-`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
-descriptor chosen by the caller. `value` is normally empty. Therefore ID
-stability depends on all three of these remaining stable:
-
-- the configured `data_path` and preservation of this database;
-- the venue/type namespace;
-- the descriptor supplied by the feed adapter.
-
-The two allocator implementations share the file and convention:
-
-- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
-  uses `securities::descriptor()`. For CME this is the Globex
-  `security_desc`, populated from the definition's symbol/tag 55.
-- `IDGenerator` (newer): derives the namespace from new domain enums and uses
-  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
-  identifier; Databento deliberately passes the decimal instrument ID, and the
-  Python bridge uses the exchange security ID.
-
-### Logical-row examples
-
-The following names are illustrative. Production uses the exact string emitted
-by the selected feed/adapter, including its case, spacing, and weekly-series
-notation.
-
-#### Legacy `SecurityIDRecorder`
-
-For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
-Globex `security_desc`. After converting the fixed security to
-`api::Security`, `securities::descriptor()` returns that same
-`glbx.security_desc()` string.
-
-| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
-|---|---|---|---|
-| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
-| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
-| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
-| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
-
-A weekly option does not get a special namespace. Standard and weekly options
-are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
-`security_desc` distinguishes the weekly series. For example, if the database
-assigned rowids `42001` through `42004` to the rows above, those integers—not
-the descriptor strings—would be the Spark security keys.
-
-Conceptually, legacy allocation is:
-
-```text
-future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
-option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
-weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
-spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
-```
-
-#### Newer `IDGenerator`
-
-The namespace result is similar, but the third argument is chosen by the
-adapter. Examples using a description-oriented adapter are:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
-// store: ns="GLBX.FUTURE", key="ESZ6"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6 C6500"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
-
-get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
-// store: ns="COINBASE.SPOT", key="BTC-USD"
-```
-
-Databento intentionally produces a different logical row even when the human
-description is `ESZ6`. If its instrument ID were `12345678`, it would call the
-equivalent of:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
-// store: ns="GLBX.FUTURE", key="12345678"
-```
-
-Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
-`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
-while a provider that sends `exchange_security_id="BTCUSD"` produces a
-different row and therefore a different Spark key.
-
-These examples show why the tuple is only logical when the adapter's naming
-contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
-describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
-
-Both enum-prefix removals currently produce the same namespace spelling (for
-example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
-rowid space. Live rows cannot share a rowid. The table does not declare
-`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
-SQLite to reuse a number later. Security-key rows should therefore be treated
-as append-only operational data.
-
-`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
-and feed latency over maximum durability against a host/power failure. The
-unique `(ns, key)` index prevents two different rows for the same logical
-security.
-
-### Other places the key is stored
-
-| Location | Purpose | Authoritative for allocation? |
-|---|---|---|
-| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
-| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
-| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
-| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
-| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
-
-The `SecurityIDRecorder::work_db()` comment says a published security is also
-recorded in the security database, but that function does **not** write
-`securities.sqlite` in the current code. Full security definitions are persisted
-separately by `SecurityCache::flush_to_disk()` through
-`db::insert_or_replace()`.
-
-## CME MDP3 / OnixS pipeline in detail
-
-### 1. Construction and thread setup
-
-`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
-`MDProducer`. It constructs the recorder with:
-
-- a lookup callback that maps `ExchangeKey` through the CME index to the
-  handle's current Spark key; and
-- `Dispatcher::current()`, which drives a 200 ms recorder timer.
-
-In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
-dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
-then calls that thread's dispatcher `run()`. The older `md run onixs` command
-uses its command/dispatcher thread similarly.
-
-The OnixS feed engine has its own configured thread pool. Instrument and market
-data callbacks therefore originate on OnixS feed-engine threads, while recorder
-database work and final key assignment execute on the MDP3 dispatcher thread.
-
-| MDP3 stage | Executing thread | DB call? |
-|---|---|---|
-| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
-| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
-| Add CME-ID index entry | Same OnixS callback thread | No |
-| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
-| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
-| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
-
-### 2. Definition callback: exchange ID is set, Spark key is not
-
-For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
-
-```cpp
-fsec.numeric_id = def.securityId();
-```
-
-It also copies symbol/descriptor/venue and pricing fields. A newly constructed
-`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
-does not change it.
-
-For a spread, every leg starts with:
-
-```cpp
-leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
-leg.key = invalid_security_key;
-```
-
-Thus both the parent and legs initially carry exchange-routing identity, but no
-Spark identity.
-
-### 3. Shared-memory allocation and exchange indexing
-
-`MDP3Listener::add_security()` performs these operations in order:
-
-1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
-   cold entry and copies the security into the cold segment.
-2. Because the key is `0`, `MDProducer` queues the handle through
-   `SecurityIDRecorder::ensure_key()`.
-3. `_index.add_security(security.numeric_id, handle)` immediately creates the
-   CME-ID-to-handle mapping.
-4. The initial exchange status is written through the handle.
-
-This ordering means incoming books can already be routed by CME ID while the
-Spark key is pending. Shared-memory consumers cannot discover the security yet.
-Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
-when `key() != 0`, so a trade received in this short window updates the cold
-last-trade state but is not added to the keyed trade log.
-
-### 4. Recorder tick: spreads, handles, then database
-
-Every 200 ms the recorder calls, in this exact order:
-
-1. `work_spreads()` tries to fill previously unresolved spread leg keys by
-   asking `CMESecurityIndex` for each leg exchange ID.
-2. `work_handles()` drains new handles. Outrights are converted to an API
-   security and queued for DB lookup immediately. A spread without all leg keys
-   is retained in `pending_handle` and its leg pointers are registered in
-   `unresolved_legs`.
-3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
-
-Because spread work precedes new-handle work, a newly seen unresolved spread
-normally waits until at least the next timer tick to resolve legs. It waits
-longer if any outright leg has not yet received its Spark key. A spread's own
-key is not requested until every leg key is non-zero.
-
-### 5. Lookup or generation
-
-`work_db()` first queries `(ns, key)`:
-
-- Existing row: use its `rowid` as the key and invoke the callback.
-- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
-  rowid inside a transaction, the row object receives that ID, and the callback
-  runs after the transaction completes.
-
-New inserts are throttled to roughly 200 per timer pass (the current `> 200`
-check can admit 201) to avoid delaying heartbeats during startup. The queues are
-TBB concurrent queues, but all SQLite access is serialized onto the recorder's
-dispatcher thread. `check_access()` asserts this single-thread rule after the
-first work pass records the dispatcher thread ID.
-
-### 6. Final assignment and visibility
-
-The queued callback captures a copy of `MDProducerHandle` and calls
-`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
-then increments `security_count`. From this point:
-
-- shared-memory scans report the security;
-- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
-- trades and RFQs can be emitted with a non-zero key;
-- resolved spreads can copy this key into their leg structures.
-
-### 7. Sunday preload is a fast path
-
-For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
-definitions from `securities.sqlite`. It converts them with
-`set_security_key=true`, so both the security key and spread leg keys come from
-the serialized API security. `MDProducer::add_security()` calls `set_key()`
-immediately, and no recorder lookup is required for that preloaded handle. The
-CME exchange ID is still restored into `numeric_id`, allowing the normal CME
-index to be built.
-
-## Generalized/newer pipeline in detail
-
-### 1. Topology and ownership
-
-`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
-`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
-`MarketDataGateway`, and starts the configured providers. For each `(venue,
-source)` returned by a provider, the gateway creates an `MdSink` containing:
-
-- a venue-local `GeneralSecurityIndex`;
-- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
-- a `GeneralMarketDataListener` connecting the source to that producer/index.
-
-The source decides its callback thread. Some sources call the listener while
-starting; others own network or replay threads. The generalized listener calls
-the producer inline on that source thread. The process-wide `IDGenerator`
-serializes its own calls with a mutex, but the listener's transient
-`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
-expected to serialize security/event callbacks.
-
-| Generalized stage | Executing thread | DB call? |
-|---|---|---|
-| Build/parse domain definition | Source/adaptor thread | No |
-| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
-| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
-| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
-| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
-| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
-
-There are two important compatibility exceptions inside the provider framework:
-
-- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
-  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
-  legacy flow documented above.
-- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
-  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
-  producer/index. Its assignment is still synchronous and pre-populated, so it
-  resembles the newer ID flow even though it does not pass through
-  `GeneralMarketDataListener`.
-
-### 2. How a new ID is constructed
-
-For a normal live generalized adapter, ID creation happens **in the adapter,
-before** `GeneralMarketDataListener::on_securities()`:
-
-```text
-logical identity = (domain venue, domain security type, adapter-chosen name)
-Spark ID         = SQLite rowid for that identity in key_ids.sqlite
-```
-
-`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
-the enum prefixes:
-
-```text
-VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
-```
-
-It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
-row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
-next rowid. That integer is wrapped in `domain::SecurityID` and placed in
-`SecurityCommon::id`.
-
-The third identity component is adapter-specific, and this matters when trying
-to reproduce a mapping:
-
-| Source/adaptor family | Typical `name` sent to `IDGenerator` |
-|---|---|
-| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
-| Taifex | product/instrument ID string |
-| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
-| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
-| Emumba Deribit | extracted Emumba security descriptor |
-| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
-
-So “security name” alone is not a universal logical key. Reproduction requires
-the exact venue enum, exact security-type enum, and the exact adapter-specific
-name used at insertion time.
-
-### 3. What `GeneralMarketDataListener` does with that ID
-
-`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
-behavior is controlled by:
-
-```cpp
-const bool use_given_sec_key = true;
-```
-
-Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
-call the DB. It copies `security_common.id.value()` into its transient
-`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
-`_add_spot()` then calls `_copy_security_common()`, which sets:
-
-```cpp
-to.key = incoming_id;
-to.numeric_id = incoming_id;
-```
-
-This collapses the generalized source/correlation ID and Spark shared-memory
-key into the same number. The source's text `exchange_sec_id` remains on the
-domain security but is not copied into `FixedSecurity::numeric_id` by this
-path.
-
-`_commit_security()` then:
-
-1. calls `MDProducer::add_security()` with an already non-zero key;
-2. causes `MDProducerHandle::set_key()` to run immediately during that call;
-3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
-   number); and
-4. returns with the security already visible in shared memory.
-
-Because the sink's producer has no recorder, supplying `id == 0` would allocate
-the shared-memory entry but never assign or publish its key. Generalized live
-sources must therefore provide a valid non-zero ID before `on_securities()`.
-
-The listener contains an alternative branch for `use_given_sec_key == false`.
-That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
-sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
-also effectively unused. Turning the flag off is not a mechanical change: an
-adapter such as Databento allocates using numeric instrument ID while the
-listener branch would use `sec.desc()`, potentially creating a second key for
-the same instrument.
-
-There is also a spread failure edge case. `_find_sec(sec, true)` records the
-parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
-ID is absent, `_add_spread()` logs and returns without committing the spread. A
-later repeat of the parent definition is then classified as a duplicate and
-ignored, so definition ordering is a hard requirement in the current code.
-
-### 4. How later books, trades, and statistics find the security
-
-Every later domain event carries a `domain::SecurityID`:
-
-- `Book::security_id()`;
-- `TradeRecord::sec_id`; or
-- `SecurityMarketDataStatistics::security_id`.
-
-The listener first uses `cc_sec_id_map` to translate that event ID to the value
-stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
-`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
-but it still serves as the admission check that the definition was seen.
-
-Most high-throughput adapters avoid SQLite on market-data events:
-
-- Databento allocates on the definition and caches
-  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
-- Taifex keeps the resolved ID in its security-details mapper.
-- Other native subscriptions maintain comparable descriptor/ID maps.
-
-The Python bridge is a notable exception: `to_bts_security_()`,
-`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
-`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
-one SQLite `SELECT` for each converted Python definition, trade, or book. It
-returns the existing ID after the first insert, but it is still a DB call and
-takes the global `IDGenerator` mutex on every event.
-
-### 5. Persistence timing and latency
-
-The newer allocator is synchronous:
-
-- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
-  thread.
-- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
-  (SQLite autocommit), also on the caller's thread.
-- Listener publication: no further DB call under the current
-  `use_given_sec_key` setting.
-
-The ID therefore exists in `key_ids.sqlite` before the adapter sends the
-definition to the listener. There is no intentional 200 ms publication delay,
-but the source callback/startup path directly pays DB and mutex latency. Bulk
-definition discovery performs one lookup and possibly one insert per security;
-`IDGenerator` does not batch those operations.
-
-WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
-not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
-occasional file-system work. The impact is usually a startup/definition cost
-when adapters cache the result, but it becomes hot-path latency for code such as
-the current Python bridge that resolves every event through the DB.
-
-`GeneralMarketDataListener` itself does not persist the full contract definition
-to `securities.sqlite`; that remains the separate security-cache/metadata
-persistence path described earlier.
-
-## Comparison: MDP3 recorder vs generalized allocator
-
-| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
-|---|---|---|
-| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
-| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
-| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
-| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
-| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
-| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
-| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
-| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
-| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
-| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
-| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
-| Batching | Yes for new rows | No |
-| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
-
-Neither algorithm generates a UUID or a content hash. Both ultimately use an
-opaque, database-assigned SQLite rowid. It is normally increasing and looks like
-a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
-used to *look up* the row is logical; the resulting number is not derivable from
-that tuple.
-
-## Can another system reconstruct the same keys?
-
-Not from `(venue, type, name)` alone. Another system can reconstruct the
-**logical lookup tuple**, but it cannot calculate the numeric Spark key because
-the key depends on the historical insertion order of every row in
-`key_ids.sqlite`.
-
-For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
-row and receive different IDs if other securities were inserted before it on
-one system. They are sequential opaque IDs, not deterministic hashes and not
-UUIDs.
-
-Ways to preserve cross-system identity are:
-
-1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
-   identical. This preserves existing numeric keys but requires coordinated
-   allocation for new ones.
-2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
-   import rows with their original rowids before either system allocates new
-   securities.
-3. Treat a stable external identity (venue + type + exchange instrument ID, or
-   another canonical instrument identifier) as the cross-system key and carry a
-   translation table to each system's local Spark key.
-4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
-   be computed independently, but collision handling, descriptor
-   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
-   with orders/recordings/databases all need explicit design.
-
-Copying only `securities.sqlite` is insufficient as an allocator strategy. It
-contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
-the mapping used to find or allocate those IDs. Conversely, copying only
-`key_ids.sqlite` preserves the numbers but not all contract metadata.
-
-## Other assignment paths in `sparkd`
-
-The two main pipelines reduce to four ways a producer handle can acquire a key:
-
-1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
-   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
-   Sunday preload, relay playback, and other replay paths use this mechanism.
-2. **Recorder-backed live input.** The input key is zero and the producer owns a
-   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
-   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
-   `numeric_id` and otherwise follows the same recorder/index design.
-3. **Service assignment.** `SyntheticSecurityService` calls
-   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
-   callback receives a copied `api::Security` with its top-level `id` set, then
-   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
-   rowid allocator.
-4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
-   never becomes public. This is not a valid production assignment path, but it
-   is the failure mode if a generalized source omits its ID.
-
-Relay and replay deserve care when reasoning about persistence:
-
-- The generalized `MDReplayer` forwards the recorded domain security ID and
-  emits later books/statistics with the recorded `sec_key`, so the listener's
-  current use-given mode preserves the recording's numeric identity.
-- The legacy relay producer converts a recorded `api::Security` with
-  `set_security_key=true`, so its shared-memory publication also preserves the
-  embedded key.
-- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
-  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
-  separate mapping population step, not a deterministic derivation of the key.
-
-## Operational implications
-
-- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
-  definition continuity matter.
-- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
-  another host, even when all instruments are identical.
-- Before moving a feed between adapters, compare the exact `name` each adapter
-  supplies to `IDGenerator`. Matching venue/type is insufficient.
-- Cache generated IDs in the source after definition processing. Avoid calling
-  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
-  this guideline.
-- Keep generated rowids below the 32-bit shared-memory ceiling until the
-  `security_key` ABI is widened or explicit bounds checking is added to the
-  generalized path.
-- Ensure spread legs are announced before generalized spread parents.
-- Treat `synchronous=0` as a durability decision. A copied DB should be taken
-  with SQLite-aware backup/checkpoint handling rather than by casually copying
-  active WAL files.
-
-Useful inspection queries (run against the configured DB path) are:
-
-```sql
-SELECT rowid AS spark_key, ns, key
-FROM store
-ORDER BY rowid;
-
-SELECT rowid AS spark_key, ns, key
-FROM store
-WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
-```
-
-## Source map
-
-- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
-  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
-  and spread-leg resolution.
-- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
-  synchronous generalized lookup/allocation and process-wide mutex.
-- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
-  location, schema, WAL/synchronous pragmas, individual inserts, and batch
-  inserts.
-- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
-  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
-  shared-memory allocation, recorder handoff, and ordered publication.
-- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
-  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
-  widths and hot/cold storage fields.
-- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
-  and
-  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
-  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
-- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
-  producer/index insertion order.
-- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
-  current dedicated MDP3 dispatcher-thread ownership.
-- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
-  and
-  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
-  generalized source/sink topology, incoming-ID mapping, fixed-security
-  conversion, and publication.
-- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
-  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
-  and
-  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
-  representative adapter-side allocation and caching behavior.
-- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
-  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
-  persistence of full security definitions.
-- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
-  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
-  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
-  translation followed by Spark-keyed market-data consumption and MBO access.
-- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
-  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
-  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
-  representative downstream use of Spark security keys.
-- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
-  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
-  ID reuse.

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -1,44 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-RUN apt-get update && apt-get install -y tzdata
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet8
-
-RUN apt-get update && apt-get install -y gdb
-RUN apt-get update && apt-get install -y fish
-RUN apt-get update && apt-get install -y vim
-RUN apt-get update && apt-get install -y less
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -1,11 +0,0 @@
-FROM ubuntu:24.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y \
-  fish \
-  less \
-  vim
-
-USER bts
-WORKDIR /home/bts

requirements.txt

@@ -1,42 +1,22 @@
 astroid==2.7.3
 autopep8==1.5.7
-blinker==1.9.0
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
-click==8.1.8
 colorama==0.4.4
-Flask==3.1.0
 idna==3.4
 isort==5.9.3
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
-MarkupSafe==3.0.2
 mccabe==0.6.1
-numpy==2.2.6
 platformdirs==2.3.0
-protobuf==6.33.4
+protobuf==4.21.12
 pybars3==0.9.7
-pybind11==3.0.1
-pybind11-stubgen==2.5.5
-pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
-pycryptodome==3.23.0
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
-websocket-client==1.8.0
-Werkzeug==3.1.3
 wrapt==1.12.1
 yapf==0.31.0

spark_client.sln

@@ -1,306 +1,103 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
 		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
+		MinSizeRel|x64 = MinSizeRel|x64
 		Release|x64 = Release|x64
-		Release|x86 = Release|x86
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
-		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
-		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
-		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,6 +8,7 @@
    public int Integer;
    public double Real;
    public string Identifier;
+   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -11,20 +11,10 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ArithmeticExpression">
     <Scanner Include="ArithmeticExpressionTokens.lex" />

src/DataStreamSuite/BaseUtilities/BaseUtilities.csproj

@@ -1,9 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
-  
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
+
 </Project>

src/DataStreamSuite/BlueRegistry/BlueRegistry.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;
 
 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
-      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -6,80 +6,23 @@ using Spark.Core.Collections;
 
 static class ArgumentComparer {
    public static int Compare(
-      IArgument value1, IArgument value2, IDictionary<string, object?> env) {
+      IArgument value1, IArgument value2, IDictionary<string,object?> env) {
       if (value1 is null) {
          throw new ArgumentNullException(nameof(value1));
       }
       if (value2 is null) {
          throw new ArgumentNullException(nameof(value2));
       }
-      var obj1 = value1.GetValue(env);
-      var obj2 = value2.GetValue(env);
-      if (obj1 is null) {
-         if (obj2 is null) {
-            return 0;
-         }
-         return -1;
-      }
-      if (obj2 is null) {
-         return 1;
-      }
-      return compareObjs(obj1, obj2);
-   }
-
-   static int compareObjs(object obj1, object obj2) {
+      var cmp1 = value1.Cast<IComparable>();
+      var cmp2 = value2.Cast<IComparable>();
       try {
-         var typ1 = obj1.GetType();
-         var typ2 = obj2.GetType();
-         bool areSameTypes = typ1 == typ2;
-         if (areSameTypes) {
-            return ((IComparable)obj1).CompareTo(obj2);
-         }
-         if (typ1.IsNumeric()) {
-            return tryCompareNumeric(obj1, obj2);
-         } else if (typ2.IsNumeric()) {
-            return -tryCompareNumeric(obj2, obj1);
-         } else if (typ1.IsEnum) {
-            if (typ2 == typeof(string)) {
-               return -((string)obj2).CompareTo(obj1.ToString());
-            }
-         } else if (typ1 == typeof(string)) {
-            return ((string)obj1).CompareTo(obj2.ToString());
-         } else if (typ1 == typeof(DateTime)) {
-            if (typ2 == typeof(TimeSpan)) {
-               return ((DateTime)obj1).Ticks.CompareTo(((TimeSpan)obj2).Ticks);
-            }
-         } else if (typ1 == typeof(TimeSpan)) {
-            if (typ2 == typeof(DateTime)) {
-               return ((TimeSpan)obj1).Ticks.CompareTo(((DateTime)obj2).Ticks);
-            }
-         }
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;
       }
-      throw new ArgumentException("Failed to compare obj1 and obj2", nameof(obj1));
-   }
-
-   static int tryCompareNumeric(object numericObj1, object obj2) {
-      var typ2 = obj2.GetType();
-      if (typ2.IsNumeric()) {
-         return new NumericComparer().Compare(numericObj1, obj2);
-      } else if (typ2.IsEnum) {
-         var undTyp2 = typ2.GetEnumUnderlyingType();
-         if (undTyp2 == typeof(byte)) {
-            return new NumericComparer().Compare(numericObj1, (byte)obj2);
-         }
-         if (undTyp2 == typeof(short)) {
-            return new NumericComparer().Compare(numericObj1, (short)obj2);
-         }
-         if (undTyp2 == typeof(int)) {
-            return new NumericComparer().Compare(numericObj1, (int)obj2);
-         }
-         if (undTyp2 == typeof(long)) {
-            return new NumericComparer().Compare(numericObj1, (long)obj2);
-         }
-      }
-      throw new ArgumentException("Failed to compare numeric obj1 and obj2", nameof(obj2));
    }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -1,11 +1,19 @@
 namespace Bts.BooleanExpressions;
 
+using bts;
+
 class BooleanExpression : IBooleanExpression {
    readonly Func<IDictionary<string, object?>, bool> evaluate;
 
    public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
       this.evaluate = evaluate;
 
-   public bool Evaluate(IDictionary<string, object?> env) =>
-      evaluate(env);
+   public bool Evaluate(IDictionary<string, object?> env) {
+      try {
+         return evaluate(env);
+      } catch (Exception ex) {
+         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
+         throw;
+      }
+   }
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -47,6 +47,11 @@ IsOfType                 '/'
 
 %%
 
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {BooleanConstant} {
    yylval.Boolean = yytext.ToLower() switch {
       "true" => true,
@@ -57,11 +62,6 @@ IsOfType                 '/'
    return (int)Token.BOOLEAN_CONSTANT;
 }
 
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {NullConstant} {
    yylval.Null = DBNull.Value;
    return (int)Token.NULL_CONSTANT;

src/DataStreamSuite/BooleanExpressions/BooleanExpressions.csproj

@@ -1,30 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="YaccLexTools" Version="1.2.3" />
+    <PackageReference Include="YaccLexTools" Version="1.2.0" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
   
   <ItemGroup>
     <Scanner Include="BooleanExpressionTokens.lex" />

src/DataStreamSuite/BooleanExpressions/GplexBuffers.cs

@@ -24,6 +24,8 @@ namespace QUT.GplexBuffers
         public BufferException(string message) : base(message) { }
         public BufferException(string message, Exception innerException)
             : base(message, innerException) { }
+        protected BufferException(SerializationInfo info, StreamingContext context)
+            : base(info, context) { }
     }
 
     internal abstract class ScanBuff
@@ -58,10 +60,12 @@ namespace QUT.GplexBuffers
             return new BuildBuffer(source);
         }
 
+#if (!BYTEMODE)
         public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
         {
             return new BuildBuffer(source, fallbackCodePage);
         }
+#endif // !BYTEMODE
 #endif // !NOFILES
     }
 
@@ -349,15 +353,17 @@ namespace QUT.GplexBuffers
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
-            NextBlk = BlockReaderFactory.Get(stream);
+            NextBlk = BlockReaderFactory.Raw(stream);
         }
 
+#if (!BYTEMODE)
         public BuildBuffer(Stream stream, int fallbackCodePage)
         {
             FileStream fStrm = (stream as FileStream);
             if (fStrm != null) FileName = fStrm.Name;
             NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
         }
+#endif
 
         /// <summary>
         /// Marks a conservative lower bound for the buffer,
@@ -428,19 +434,43 @@ namespace QUT.GplexBuffers
     //
     internal static class BlockReaderFactory
     {
+        public static BlockReader Raw(Stream stream)
+        {
+            return delegate(char[] block, int index, int number)
+            {
+                byte[] b = new byte[number];
+                int count = stream.Read(b, 0, number);
+                int i = 0;
+                int j = index;
+                for (; i < count; i++, j++)
+                    block[j] = (char)b[i];
+                return count;
+            };
+        }
 
-        public static BlockReader Get(Stream stream, int fallbackCodePage = -2)
+#if (!BYTEMODE)
+        public static BlockReader Get(Stream stream, int fallbackCodePage)
         {
             Encoding encoding;
             int preamble = Preamble(stream);
 
             if (preamble != 0)  // There is a valid BOM here!
                 encoding = Encoding.GetEncoding(preamble);
-            else if (fallbackCodePage >= 0) // Anything but "guess"
+            else if (fallbackCodePage == -1) // Fallback is "raw" bytes
+                return Raw(stream);
+            else if (fallbackCodePage != -2) // Anything but "guess"
                 encoding = Encoding.GetEncoding(fallbackCodePage);
             else // This is the "guess" option
-                encoding = GuessEncoding(stream);
-
+            {
+                int guess = new Guesser(stream).GuessCodePage();
+                stream.Seek(0, SeekOrigin.Begin);
+                if (guess == -1) // ==> this is a 7-bit file
+                    encoding = Encoding.ASCII;
+                else if (guess == 65001)
+                    encoding = Encoding.UTF8;
+                else             // ==> use the machine default
+                    encoding = Encoding.Default;
+            }
             StreamReader reader = new StreamReader(stream, encoding);
             return reader.Read;
         }
@@ -458,14 +488,6 @@ namespace QUT.GplexBuffers
             int b2 = stream.ReadByte();
             if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                 return 65001; // UTF8
-
-            int b3 = stream.ReadByte();
-            // Check for UTF-32 BOM (Byte Order Mark)
-            if (b0 == 0x00 && b1 == 0x00 && b2 == 0xFE && b3 == 0xFF)
-                return 12001; // UTF-32BE
-            if (b0 == 0xFF && b1 == 0xFE && b2 == 0x00 && b3 == 0x00)
-                return 12000; // UTF-32LE
-
             //
             // There is no unicode preamble, so we
             // return denoter for the machine default.
@@ -473,150 +495,7 @@ namespace QUT.GplexBuffers
             stream.Seek(0, SeekOrigin.Begin);
             return 0;
         }
-
-        private static Encoding GuessEncoding(Stream stream)
-        {
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf16(stream, out Encoding encoding))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return encoding;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsUtf8(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.UTF8;
-            }
-
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsAscii(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.ASCII;
-            }
-
-#if !NET462_OR_GREATER
-            stream.Seek(0, SeekOrigin.Begin);
-            if (IsLatin1(stream))
-            {
-                stream.Seek(0, SeekOrigin.Begin);
-                return Encoding.Latin1;
-            }
-#endif
-            stream.Seek(0, SeekOrigin.Begin);
-            return Encoding.Default;
-        }
-
-        private static bool IsUtf8(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                if ((b & 0x80) == 0) // 0xxxxxxx
-                {
-                    continue;
-                }
-                else if ((b & 0xE0) == 0xC0) // 110xxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF0) == 0xE0) // 1110xxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else if ((b & 0xF8) == 0xF0) // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                {
-                    if (!IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream) || !IsValidUtf8ContinuationByte(stream)) return false;
-                }
-                else
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsValidUtf8ContinuationByte(Stream stream)
-        {
-            int b = stream.ReadByte();
-            return b != -1 && (b & 0xC0) == 0x80; // 10xxxxxx
-        }
-
-        private static bool IsUtf16(Stream stream, out Encoding encoding)
-        {
-            encoding = Encoding.Default;
-
-            if (stream.Length >= 2)
-            {
-                // Read the first two bytes
-                int b0 = stream.ReadByte();
-                int b1 = stream.ReadByte();
-
-                // Check for UTF-16 BOM (Byte Order Mark)
-                if (b0 == 0xFF && b1 == 0xFE)
-                {
-                    encoding = Encoding.Unicode;
-                    return true; // UTF-16LE
-                }
-                if (b0 == 0xFE && b1 == 0xFF)
-                {
-                    encoding = Encoding.BigEndianUnicode;
-                    return true; // UTF-16BE
-                }
-
-                // Reset the stream position
-                stream.Seek(0, SeekOrigin.Begin);
-
-                // Check for common UTF-16 patterns
-                byte[] buffer = new byte[2];
-                while (stream.Read(buffer, 0, 2) == 2)
-                {
-                    if (buffer[0] == 0x00 && buffer[1] != 0x00)
-                    {
-                        encoding = Encoding.BigEndianUnicode;
-                        return true; // Likely UTF-16BE
-                    }
-                    if (buffer[0] != 0x00 && buffer[1] == 0x00)
-                    {
-                        encoding = Encoding.Unicode;
-                        return true; // Likely UTF-16LE
-                    }
-                }
-            }
-
-            return false;
-        }
-
-        private static bool IsAscii(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0x7F)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static bool IsLatin1(Stream stream)
-        {
-            int b;
-            while ((b = stream.ReadByte()) != -1)
-            {
-                // Latin1 includes byte values from 0x00 to 0xFF
-                if (b > 0xFF)
-                {
-                    return false;
-                }
-            }
-            return true;
-        }
-
+#endif // !BYTEMODE
     }
 #endif // !NOFILES
     #endregion Buffer classes
@@ -661,6 +540,180 @@ namespace QUT.GplexBuffers
             return 0;
         }
     }
+#region guesser
+#if (!BYTEMODE)
+    // ==============================================================
+    // ============          Encoding Guesser           =============
+    // ==============================================================
+
+    /// <summary>
+    /// This class provides a simple finite state automaton that
+    /// scans the file looking for (1) valid UTF-8 byte patterns,
+    /// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
+    /// The method then guesses whether it is UTF-8 or maybe some 
+    /// local machine default encoding.  This works well for the
+    /// various Latin encodings.
+    /// </summary>
+    internal class Guesser
+    {
+        ScanBuff buffer;
+
+        public int GuessCodePage() { return Scan(); }
+
+        const int maxAccept = 10;
+        const int initial = 0;
+        const int eofNum = 0;
+        const int goStart = -1;
+        const int INITIAL = 0;
+        const int EndToken = 0;
+
+        #region user code
+        /* 
+         *  Reads the bytes of a file to determine if it is 
+         *  UTF-8 or a single-byte code page file.
+         */
+        public long utfX;
+        public long uppr;
+        #endregion user code
+
+        int state;
+        int currentStart = startState[0];
+        int code;
+
+        #region ScannerTables
+        static int[] startState = new int[] { 11, 0 };
+
+        #region CharacterMap
+        static sbyte[] map = new sbyte[256] {
+/*     '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*      'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+/*   '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 
+/*   '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
+/*   '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 
+/*   '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
+        #endregion
+
+        static sbyte[][] nextState = new sbyte[][] {
+            new sbyte[] {0, 0, 0, 0, 0, 0},
+            new sbyte[] {-1, -1, 10, -1, -1, -1},
+            new sbyte[] {-1, -1, -1, -1, -1, -1},
+            new sbyte[] {-1, -1, 8, -1, -1, -1},
+            new sbyte[] {-1, -1, 5, -1, -1, -1},
+            new sbyte[] {-1, -1, 6, -1, -1, -1},
+            new sbyte[] {-1, -1, 7, -1, -1, -1},
+            null,
+            new sbyte[] {-1, -1, 9, -1, -1, -1},
+            null,
+            null,
+            new sbyte[] {-1, 1, 2, 3, 4, 2}
+        };
+
+
+        [SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
+        // Reason for suppression: cannot have self-reference in array initializer.
+        static Guesser()
+        {
+            nextState[7] = nextState[2];
+            nextState[9] = nextState[2];
+            nextState[10] = nextState[2];
+        }
+
+        int NextState()
+        {
+            if (code == ScanBuff.EndOfFile)
+                return eofNum;
+            else
+                return nextState[state][map[code]];
+        }
+        #endregion
+
+        public Guesser(System.IO.Stream file) { SetSource(file); }
+
+        public void SetSource(System.IO.Stream source)
+        {
+            this.buffer = new BuildBuffer(source);
+            code = buffer.Read();
+        }
+
+        int Scan()
+        {
+            for (; ; )
+            {
+                int next;
+                state = currentStart;
+                while ((next = NextState()) == goStart)
+                    code = buffer.Read();
+
+                state = next;
+                code = buffer.Read();
+
+                while ((next = NextState()) > eofNum)
+                {
+                    state = next;
+                    code = buffer.Read();
+                }
+                if (state <= maxAccept)
+                {
+                    #region ActionSwitch
+#pragma warning disable 162
+                    switch (state)
+                    {
+                        case eofNum:
+                            switch (currentStart)
+                            {
+                                case 11:
+                                    if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
+                                    else if (uppr * 10 > utfX) return 0;   /* default code page */
+                                    else return 65001;                     /* UTF-8 encoding */
+                                    break;
+                            }
+                            return EndToken;
+                        case 1: // Recognized '{Upper128}',	Shortest string "\xC0"
+                        case 2: // Recognized '{Upper128}',	Shortest string "\x80"
+                        case 3: // Recognized '{Upper128}',	Shortest string "\xE0"
+                        case 4: // Recognized '{Upper128}',	Shortest string "\xF0"
+                            uppr++;
+                            break;
+                        case 5: // Recognized '{Utf8pfx4}{Utf8cont}',	Shortest string "\xF0\x80"
+                            uppr += 2;
+                            break;
+                        case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}',	Shortest string "\xF0\x80\x80"
+                            uppr += 3;
+                            break;
+                        case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}',	Shortest string "\xF0\x80\x80\x80"
+                            utfX += 3;
+                            break;
+                        case 8: // Recognized '{Utf8pfx3}{Utf8cont}',	Shortest string "\xE0\x80"
+                            uppr += 2;
+                            break;
+                        case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}',	Shortest string "\xE0\x80\x80"
+                            utfX += 2;
+                            break;
+                        case 10: // Recognized '{Utf8pfx2}{Utf8cont}',	Shortest string "\xC0\x80"
+                            utfX++;
+                            break;
+                        default:
+                            break;
+                    }
+#pragma warning restore 162
+                    #endregion
+                }
+            }
+        }
+    } // end class Guesser
+    
+#endif // !BYTEMODE
+#endregion
 #endif // !NOFILES
 
 // End of code copied from embedded resource

src/DataStreamSuite/Bts.ColumnExpressions/Bts.ColumnExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnExpression">
     <Scanner Include="ColumnExpressionTokens.lex" />

src/DataStreamSuite/Bts.ColumnSortExpressions/Bts.ColumnSortExpressions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -10,20 +10,11 @@
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="DataStreamCore" Version="$(SparkVersion)" />
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnSortExpression">
     <Scanner Include="ColumnSortExpressionTokens.lex" />

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -1,27 +1,21 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ExpressionCore" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+  </ItemGroup>
 
   <ItemGroup Label="ColumnMapExpression">
     <Scanner Include="ColumnMapExpressionTokens.lex" />

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
-      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
-         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CSVReaderStream.cs

@@ -1,18 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("CSVReaderStream")]
-   public class CSVReaderStream : ViewStream {
-      public CSVReaderStream(string file_name)
-         : this(file_name, false) { }
-      public CSVReaderStream(string file_name, bool guess_schema) {
-         ViewStream rs = new BCG.CSVReaderStream(file_name);
-         if (guess_schema) {
-            rs = new SchemaGuesserStream(rs);
-         }
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CSVSaverStream.cs

@@ -1,13 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("CSVSaverStream")]
-   public class CSVSaverStream: ViewStream {
-      public CSVSaverStream(ViewStream parent_str, string file_name) {
-         BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,43 +0,0 @@
-namespace BCG {
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
-      IDataPacket? myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
-      private int myId = 0;
-      private ViewStream myVS;
-
-      public CountStream(ViewStream vs) {
-         myVS = vs;
-      }
-
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
-
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
-
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
-            myLastFiredDataPacket =
-               myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
-                  :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  myLastFiredDataPacket.newRows)
-            );
-      }
-
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
-         retVal.addRow(new BlueRow(mySchema, id, ((IDataPacket)dp).newRows.Count));
-         return retVal;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using bts;
-
-   using BCG;
-
-   using System;
-   [DataStreamUserRespresentation("DateTimeExpanderStream")]
-   public class DateTimeExpanstionStream : ViewStream {
-      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
-         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
-            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_year");
-            return rv;
-         };
-         ViewStream rs = new MapStream(parent_stream,
-         s => get_schema(s),
-         r => {
-            DateTime dt = r.get_datetime(inp_col);
-            IBlueTableSchema s = get_schema(r.schema);
-            MutableBlueRow rv = new MutableBlueRow(s, r);
-            rv.setValue($"{inp_col}_date", dt.Date);
-            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
-            rv.setValue($"{inp_col}_hour", dt.Hour);
-            rv.setValue($"{inp_col}_minute", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
-            rv.setValue($"{inp_col}_year", dt.Date.Year);
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -1,164 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Bts.DataStreamCore;
-
-   using System;
-   using System.Text.RegularExpressions;
-
-   [DataStreamUserRespresentation("FilterStream")]
-   public class FilterStream : ViewStream {
-      public class RegExpFilter : IStreamFilter {
-         private Regex[] myRegExps;
-         private string[] myCols;
-
-         public RegExpFilter(string col, string regExp)
-            : this(new string[] { col }, new string[] { regExp }) { }
-
-         public RegExpFilter(string[] cols, string[] regExps) {
-            myCols = cols;
-            myRegExps = new Regex[regExps.Length];
-            for (int i = 0; i < regExps.Length; i++)
-               myRegExps[i] = new Regex(regExps[i]);
-         }
-
-         #region IStreamFilter Members
-         public bool filter(IBlueRow dr) {
-            for (int i = 0; i < myCols.Length; i++) {
-               var val = dr[myCols[i]];
-               var str = val is DBNull
-                  ? string.Empty
-                  : (val?.ToString() ?? string.Empty);
-               if (!myRegExps[i].IsMatch(str)) {
-                  return false;
-               }
-            }
-            return true;
-         }
-         #endregion
-      }
-      BlueTable? myLastFiredRows;
-      IBlueRowCollection? myLastReceivedRows;
-      private IStreamFilter myFilter;
-      private ViewStream myParentStream;
-      private string myFilterString;
-
-      #region constructors
-      public delegate bool OnFilter(IBlueRow r);
-      private class OnFilterWrapper : IStreamFilter {
-         public OnFilterWrapper(OnFilter f) { this.f = f; }
-
-         public bool filter(IBlueRow dr) { return f(dr); }
-
-         private OnFilter f;
-      }
-      public FilterStream(ViewStream vs, OnFilter f)
-         : this(vs, new OnFilterWrapper(f)) { }
-
-
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
-         myParentStream = vs;
-         myFilter = filter;
-         myFilterString = filter.ToString() ?? string.Empty;
-         if (filter is IDynamicStreamFilter dynFilter) {
-            dynFilter.onFilterChange += myHandleFilterChange;
-         }
-      }
-
-      public FilterStream(ViewStream vs, string filterExpression)
-         : this(vs, new BooleanExpressionFilter(filterExpression)) {
-         myFilterString = filterExpression;
-      }
-
-      public FilterStream(ViewStream vs, string col, string regExp)
-         : this(vs, new RegExpFilter(col, regExp)) {
-         myFilterString = col + " , " + regExp;
-      }
-
-      public FilterStream(ViewStream vs)
-         : this(vs, new BinaryFilter(true)) {
-         myFilterString = true.ToString();
-      }
-      #endregion
-
-      #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
-
-      protected override void myOnLastSubscriber() {
-         myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
-      }
-
-      protected override void myFireEvent(object o) {
-         var dp = (IDataPacket)o;
-         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
-            base.myFireEvent(o);
-      }
-      #endregion
-
-      private void myHandleCallbacks(object newValue) {
-         DataPacket retVal;
-         IDataPacket arrivedPacket = (IDataPacket)newValue;
-         myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null) {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
-            foreach (IBlueRow r in arrivedPacket.deletedRows) {
-               if (myLastFiredRows.containsID(r.Id())) {
-                  myLastFiredRows.deleteRow(r);
-                  deletedRows.Value.addRow(r);
-               }
-            }
-            var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
-            foreach (IBlueRow r in arrivedPacket.addedRows) {
-               if (myFilter.filter(r)) {
-                  myLastFiredRows.addRow(r);
-                  addedRows.Value.addRow(r);
-               }
-            }
-         } else {
-            myLastFiredRows = new BlueTable(arrivedPacket.Schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
-      }
-
-      public IStreamFilter filter {
-         set {
-            {
-               myFilter = value;
-               if (numberOfSubscribers > 0) {
-                  if (myLastReceivedRows != null) {
-                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
-                     foreach (IBlueRow r in myLastReceivedRows)
-                        if (myFilter.filter(r))
-                           nRows.addRow(r);
-                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
-                        aRows = new BlueTable(nRows.schema);
-                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
-                     myLastFiredRows = nRows;
-                     myFireEvent(new DataPacket(nRows, aRows, dRows));
-                  }
-               }
-            }
-         }
-         get {            
-               return myFilter;
-         }
-      }
-
-      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
-
-      public override string ToString() {
-         return GetType().ToString() + " " + myFilterString;
-      }
-
-      private void myHandleFilterChange() {
-         filter = myFilter;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -1,142 +0,0 @@
-using BCG;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("GroupByStream")]
-   public class GroupByStream : ViewStream {
-
-      public interface INonIncGroupByEvaluator {
-         // grouped_rows
-         IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
-         string[]? get_group_by_cols();
-      }
-
-      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
-         Func<string[]?> _groupby_cols_func;
-         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
-         public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
-                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
-            _groupby_cols_func = groupby_cols_func;
-            _evaluate_func = evaluate_func;
-         }
-
-         public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
-            BlueTable? rv = null;
-            foreach (var kv in grouped_rows) {
-               List<IBlueRow> g_rows = kv.Value;
-               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
-               if (out_rows != null && out_rows.Count > 0) {
-                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
-               }
-               if (rv != null)
-                  rv.addRows(out_rows);
-            }
-            return rv;
-         }
-
-         public string[]? get_group_by_cols() {
-            return _groupby_cols_func();
-         }
-
-      }
-      ViewStream? _main_stream;
-      INonIncGroupByEvaluator? _groupby_func;
-      BlueRowHasher? _row_hasher;
-      string[]? _groupby_cols;
-      private Stopwatch _timer = new();
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      IDisposable? subscription;
-
-      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
-         _main_stream = main_stream;
-         _groupby_func = groupby_func;
-         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
-      }
-
-      public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
-                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
-                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
-
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
-      : this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
-
-      // this is the main constructor that does everything
-      public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
-         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            // this si where the group by aggregation happens
-            IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
-            if (rv == null)
-               return new List<IBlueRow>();
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-
-         string[]? groupby_cols_func() {
-            if (string.IsNullOrEmpty(groupby_cols)) {
-               return null;
-            }
-            return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
-         }
-      }
-      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
-      protected override void myOnFirstSubscriber() {
-         Log.isnull(subscription);
-         subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
-      }
-
-      protected override void myOnLastSubscriber() {
-         subscription?.Dispose();
-         subscription = null;
-      }
-
-      private void _main_stream_onchange(object nv) {
-         Log.notnull(_groupby_func);
-         var dp = (IDataPacket)nv;
-         ///grouping together
-         _timer.Restart();
-         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
-         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
-         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
-         /// reducing
-         _timer.Restart();
-         IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
-         if (total_state == null)
-            // mayeb returna n empty table ?
-            return;
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
-         _n_total_fires++;
-         if (applyElapsedMs >= 50) {
-            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         var deletedRows = myLastValue is null
-            ? new BlueTable(total_state.schema)
-            : ((IDataPacket)myLastValue).newRows;
-         myFireEvent(new DataPacket(total_state, total_state, deletedRows));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,204 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Bts.DataStream;
-
-   using Spark.Core.Diagnostics;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Diagnostics.CodeAnalysis;
-   using System.Linq;
-
-   using RowHash = System.Int32;
-
-   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
-         // initialized in cons
-         readonly string[] _join_col_names;
-         private readonly BlueRowHasher _hasher;
-         // initialized on first udpate         
-         IBlueTableSchema? _schema;
-         int[]? _join_cols;
-         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
-
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
-            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
-            _hasher = row_hasher;
-         }
-
-         [NotNullIfNotNull(nameof(_schema))]
-         public IBlueTableSchema? schema => _schema;
-
-         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
-
-         [MemberNotNull(nameof(_schema))]
-         public IBlueTableSchema RequireSchema() =>
-            ExceptionAssert.MemberNotNull(_schema);
-
-         public List<IBlueRow>? get_matching_rows(int hash) =>
-            _hash_row_map.GetValueOrDefault(hash);
-
-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) {
-               return 0;
-            }
-            return _hasher.get_hash(r, _join_cols);
-         }
-
-         public void update_data(IDataPacket dp) {
-            if (!ready) {
-               _schema = dp.newRows.schema;
-               _join_cols = _schema.ResolveIndices(_join_col_names);
-            }
-            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
-               int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
-                  hash_rows = new List<IBlueRow>();
-                  _hash_row_map.Add(hash, hash_rows);
-               }
-               hash_rows.Add(r);
-            }
-         }
-
-         [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready {
-            get {
-               return _schema != null;
-            }
-         }
-
-      }
-
-      private ViewStream _ls, _rs;
-      private JoinBookkeepingData _ldata, _rdata;
-      IBlueTableSchema? _schema;
-
-      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
-
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
-         _ls = ls;
-         _rs = rs;
-         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
-         Log.assert(lj_col_list.Count == rj_col_list.Count);
-         int n_cols = lj_col_list.Count;
-         BlueRowHasher hasher = new BlueRowHasher(n_cols);
-         _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
-         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _ls.onChange += _ls_onchange;
-         _rs.onChange += _rs_onchange;
-      }
-
-      private void _ls_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         _ldata.update_data(dp);
-         _form_and_fire();
-      }
-
-      private void _rs_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         _rdata.update_data(dp);
-         _form_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         _ls.onChange -= _ls_onchange;
-         _rs.onChange -= _rs_onchange;
-      }
-
-      [MemberNotNull(nameof(_schema))]
-      private void _build_schema() {
-         if (_schema != null)
-            return;
-         var lschema = Log.notnull(_ldata.schema);
-         var rschema = Log.notnull(_rdata.schema);
-         var cis = new List<ColumnInfo> {
-            new("id", 0, TypeCode.Int32)
-         };
-         foreach (ColumnInfo ci in lschema.columnInfo) {
-            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         foreach (ColumnInfo ci in rschema.columnInfo) {
-            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
-            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
-         }
-         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
-      }
-
-      private void _form_and_fire() {
-         if (!_ldata.ready || !_rdata.ready)
-            return;
-         _build_schema();
-         List<object[]> res_rows = new List<object[]>();
-
-         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
-            RowHash hash = kv.Key;
-            List<IBlueRow> l_rows = kv.Value;
-            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
-
-            foreach (IBlueRow lr in l_rows) {
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
-                     object[] rowvals = build_row_array(res_rows.Count, lr);
-                     copy_row(rowvals, r, false);
-                     res_rows.Add(rowvals);
-                  }
-               } else {
-                  object[] rowvals = build_row_array(res_rows.Count, lr);
-                  res_rows.Add(rowvals);
-               }
-            }
-         }
-         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
-            rv.addRow(new BlueRow(_schema, rval));
-         }
-         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
-         myFireEvent(dp);
-
-         object[] build_row_array(int id, IBlueRow l_row) {
-            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
-               rv[i] = DBNull.Value;
-            }
-            rv[0] = id;
-            copy_row(rv, l_row, true);
-            return rv;
-         }
-
-         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
-            int offset = 1;
-            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
-               offset = 1;
-               sdata = _ldata;
-            } else {
-               sdata = _rdata;
-               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
-            }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
-               rval[offset + i] = r[i];
-            }
-         }
-      }
-   }
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -1,28 +0,0 @@
-//namespace bts.Blueorb {
-//   using BCG;
-
-//   public class JsonSaverStream : ViewStream {
-//      private ViewStream _stream;
-//      private string _file_name;
-//      public JsonSaverStream(ViewStream vs, string file_name) {
-//         _stream = vs;
-//         _file_name = file_name;
-//      }
-
-//      protected override void myOnFirstSubscriber() {
-//         _stream.onChange += _on_data;
-//      }
-
-//      private void _on_data(object dp_obj) {
-//         if (dp_obj == null) {
-//            return;
-//         }
-//         DataPacket dp = dp_obj as DataPacket;
-//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
-//      }
-
-//      protected override void myOnLastSubscriber() {
-//         _stream.onChange -= _on_data;
-//      }
-//   }
-//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -1,91 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-
-using Amazon.Runtime.Internal.Util;
-
-using BCG;
-using BCG.Utilities;
-
-using bts;
-
-namespace bts.BlueOrb{
-   public class LiteralStream : ViewStream {
-
-      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
-         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
-         string[] pairs = key_value_pairs.Split(' ');
-         foreach (string p in pairs) {
-            string[] kvp = p.Split(':');
-            string col_name = kvp[0];
-            string val = kvp[1];
-            var tup = BlueDataUtility.guess_type(val);
-            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
-            rv.Add(Tuple.Create(ci, tup.Item2));
-         }
-         return rv;
-      }
-
-      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
-      // col_name: col_value ... ; col_name: col_value
-      public LiteralStream(string table_name, string rows) {
-         Func<string, int, IBlueRow> get_row = (json_row, id) => {
-            Dictionary<string, object> map = new Dictionary<string, object>();
-            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
-            //Logger.assert(col_vals.Length == 2);
-            foreach (string t in toks) {
-               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
-               string col = col_vals[0];
-               string val = col_vals[1];
-               map.Add(col, val);
-            }
-            map["id"] = id;
-            return new BlueRow(map);
-         };
-         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
-         BlueTable table = null;
-         foreach (string json_string in row_arr) {
-            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
-            get_row(json_string, table == null ? 0 : table.Count);
-            table = table ?? new BlueTable(r.schema);
-            table.addRow(r);
-         }
-         ViewStream str = new BlueTableStream(table);
-         str = new SchemaGuesserStream(str);
-         mySetRealSource(str);
-      }
-
-      public LiteralStream(string tableName, string schema, string value) {
-         string[] columns = schema.Split(' ');
-
-
-         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id", 0, typeof(int));
-         int n = 1;
-         foreach (string c in columns) {
-            string[] columnAndType = c.Split('/');
-            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c, n, typeof(double));
-            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
-            n++;
-         }
-
-
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
-         string[] rows = value.Split(';');
-         n = 0;
-         foreach (string r in rows)
-            try {
-               if (r == string.Empty)
-                  continue;
-               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
-            } catch (Exception e) {
-               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
-            }
-
-         mySetRealSource(new BlueRowCollectionStaticStream(t));
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -1,89 +0,0 @@
-using System;
-
-using BCG.Utilities.General;
-using BCG;
-using bts;
-
-namespace bts.BlueOrb {
-   [DataStreamUserRespresentation("MillisecondStream")]
-   public class MillisecondStream : ViewStream, IWorker {
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private readonly TimeSpan _period;
-      private readonly int _size;
-      private readonly ViewStreamDatabase _vsd;
-      private IBlueTableSchema _schema;
-      private bts.CircularBufferTable _table;
-
-      public MillisecondStream(int interval_msecs, int size)
-      : this(ViewStreamDatabase.instance, interval_msecs, size) {
-      }
-
-      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
-         _vsd = vsd;
-         _period = TimeSpan.FromMilliseconds(interval_msecs);
-         _size = size;
-         _schema = BlueDataUtility.form_schema("Milliseconds",
-               Tuple.Create("id", TypeCode.Int32),
-               Tuple.Create("time", TypeCode.DateTime),
-               Tuple.Create("display_time", TypeCode.String));
-         _table = new CircularBufferTable(_schema, _size);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private int _id = 0;
-      public void doWork() {
-         if (DateTime.Now - _last_fire_time >= _period) {
-            _last_fire_time = DateTime.Now;
-            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
-            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
-            _table.add_row(nr);
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-            myFireEvent(dp);
-         }
-      }
-   }
-   internal class MSecondStream : TimeUnitStream {
-      public MSecondStream(int intervalInms, int size)
-         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
-
-      protected override DateTime myRoundTime(DateTime dt) { return dt; }
-   }
-
-   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
-      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
-                                                               new ColumnInfo("id", 0, typeof(int)),
-                                                               new ColumnInfo("time", 1, typeof(string)));
-      public DisplayableMillisecondStream(int intervalInMs, int sz) {
-         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
-      }
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         DateTime time = row.get_datetime("time");
-         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
-         return new BlueRow(mySchema, row["id"], time_txt);
-      }
-
-      #endregion
-
-      private string myToString(DateTime dt) {
-         string sep = ":";
-         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -1,168 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   //using MathNet.Numerics.Statistics;
-
-
-   public enum MultiRowOp {
-      Mean,
-      Median,
-      Mode,
-      Range,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum,
-      EdgeDiff,
-      EdgeDiffPercentage,
-   }
-   public class MultiRowOpHelper {
-      // input + op
-      public readonly List<Tuple<string, MultiRowOp>> operations;
-
-      public static MultiRowOpHelper parse_ops(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
-         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new MultiRowOpHelper(rv);
-      }
-
-      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
-         operations = operations_;
-      }
-
-      // aggregates multiple rows into one - ie does group_by compute
-      public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
-         if (rows.Count == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      // aggregates multiple rows into one, used by MultiRowRollingMathStream
-      public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
-         if (rows.Count == 0) {
-            return null;
-         }
-         IBlueRow ir = rows.ElementAt(start_idx);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         // out_col vs inp_cal + type
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-            or.setValue("id", ir["id"]);
-         }
-         return or;
-      }
-
-      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
-      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-
-      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
-         if (rolling_rows.Count == 0) {
-            return null;
-         }
-         IBlueRow ir = rolling_rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
-            new Dictionary<string, Tuple<string, MultiRowOp>>();
-         foreach (var t in operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            MultiRowOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
-         switch (op) {
-            case MultiRowOp.Max:
-               return col_vals.Max<double>();
-            case MultiRowOp.Min:
-               return col_vals.Min<double>();
-            case MultiRowOp.Mean:
-               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
-            case MultiRowOp.Sum:
-               return col_vals.Sum();
-            case MultiRowOp.Median:
-               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
-            //case MultiRowOp.Mode:
-            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
-            case MultiRowOp.First:
-               return col_vals.First();
-            case MultiRowOp.Last:
-               return col_vals.Last();
-            case MultiRowOp.StdDev:
-               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
-            case MultiRowOp.EdgeDiff:
-               return col_vals.First() - col_vals.Last();
-            case MultiRowOp.EdgeDiffPercentage: {
-                  double first = col_vals.First();
-                  double last = col_vals.Last();
-                  return ((first - last) / last) * 100.0;
-               }
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -1,40 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-
-   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
-   public class MultiRowRollingMathStream : ViewStream {
-      private ViewStream _sorted_parent_str;
-      private MultiRowOpHelper _op;
-      private int _rolling_window_sz;
-      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
-         _sorted_parent_str = new SortStream(parent_str, sort_exp);
-         _op = MultiRowOpHelper.parse_ops(op_txt);
-         _rolling_window_sz = rolling_window_sz;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _sorted_parent_str.onChange += _parent_str_onchange;
-      }      
-
-      protected override void myOnLastSubscriber() {
-         _sorted_parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         var dp = (IDataPacket)dp_obj;
-         var rows = dp.newRows.to_list();
-         BlueTable? out_table = null;
-         for (int i = 0; i < rows.Count; i++) {
-            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
-               ?? throw new InvalidOperationException("group_by_compute returned null row");
-            if (out_table == null)
-               out_table = new BlueTable(out_r.schema);
-            out_table.addRow(out_r);
-         }            
-         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
-         myFireEvent(dp_out);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,190 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using Spark.Core.Diagnostics;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   #region mappings
-   internal class ProjectData {
-      // schema from the base stream
-      public readonly IBlueTableSchema in_schema;
-      // this is the projected schema
-      public readonly IBlueTableSchema out_schema;
-      // projected cols to the input cols, for cols who's names are changed
-      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
-      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
-                  IDictionary<string, string> mapped_to_orig_col_map_) {
-         in_schema = in_schema_;
-         out_schema = out_schema_;
-         mapped_to_orig_col_map = mapped_to_orig_col_map_;
-      }
-   }
-   #endregion
-
-
-#region Row with mappings build in
-   internal class NewProjectRow : IBlueRow {
-      #region IBlueRow members
-      private ProjectData _project_data;
-      private IBlueRow _row;
-      public NewProjectRow(ProjectData project_data, IBlueRow row) {
-         _project_data = project_data;
-         _row = row;
-      }
-      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
-      #endregion
-
-      #region ICollection members
-      public int Count { get { return schema.columnNames.Length; } }
-
-      public bool IsSynchronized { get { return false; } }
-
-      public object SyncRoot { get { return this; } }
-
-      public void CopyTo(Array array, int index) {
-         int i = index;
-         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
-      }
-      #endregion
-
-      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
-
-      public bool IsReadOnly { get { return true; } }
-
-      public object? this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            return this[(int)key];
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public object? this[int index] {
-         get {
-            return this[schema.columnInfo[index].colName];
-         }
-      }
-
-      public object? this[string col_name] {
-         get {
-            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
-               return _row[mapped_to_col];
-            }
-            return _row[col_name];
-         }
-      }
-
-      public ICollection Keys { get { return schema.columnNames; } }
-
-      public ICollection Values {
-         get {
-            object[] retVal = new object[schema.columnNames.Length];
-            CopyTo(retVal, 0);
-            return retVal;
-         }
-      }
-
-      public void Add(object key, object? value) =>
-         throw new NotSupportedException();
-
-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public bool Contains(object key) { return schema[(string)key] != null; }
-
-      /// <summary>
-      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
-      /// iterate through the values, typecast this into an IEnumerable
-      /// </summary>
-      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-      #endregion
-   }
-   #endregion
-
-   [DataStreamUserRespresentation("ProjectStream")]
-   public class NewProjectStream : ViewStream, IStreamMapper {
-
-      private ViewStream _stream;
-      private string _project_expression;
-      ProjectData? _project_data;
-      //public ProjectData 
-      public NewProjectStream(ViewStream vs, string project_expression) {
-         _stream = vs;
-         _project_expression = project_expression;
-         _stream = new MapStream(vs, this);
-         mySetRealSource(_stream);
-      }
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
-         _project_data = _create_project_data(input_schema, _project_expression);
-         return _project_data.out_schema;
-      }
-
-      public IBlueRow map(IBlueRow row) =>
-         new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
-
-      // this is where most of the logic is, in terms of translating cols
-      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
-                                                      string project_expression) {
-         project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ', ',');
-         HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
-         // by default every col will show up, so if we have the projection as (a:a1)
-         // it will convert a to a1, 
-         // modifications (6/30/2025):
-         // only the columns represented here, either mapped
-         // or not, will show
-         // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa b") -> aa b
-         foreach (string o_p_tok in project_toks) {
-            string p_tok = o_p_tok.Trim();
-            if (p_tok.Contains(":")) {
-               // explicit mapping
-               string[] map_toks = p_tok.Split(":");
-               string orig_col = map_toks[0];
-               string mapped_col = map_toks[1];
-               orig_to_mapped.Add(orig_col, mapped_col);  
-            } else {
-               // this is a col_name explicitely stated
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
-            }
-         }
-
-         // make the new schema
-         List<ColumnInfo> new_cols = new List<ColumnInfo>();
-         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;           
-            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
-               string mapped_name = orig_to_mapped[ci.colName];
-               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-            }
-         }
-         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
-         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
-         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
-      }
-
-   
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -1,128 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   using System.IO;
-   using System.Diagnostics;
-   using System;
-
-   [DataStreamUserRespresentation("PythonDataframeStream")]
-   public class PythonDataframeStream : ViewStream {
-      private ViewStream _parent_str;
-      private readonly string _in_csv_file_name;
-      private readonly string _out_csv_filename;
-      private readonly string _transform_function;
-      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
-         var guid = System.Guid.NewGuid();
-         _in_csv_file_name = $"pdfs_in_{guid}.csv";
-         _out_csv_filename = $"pdfs_out_{guid}.csv";
-         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
-         _transform_function = transform_function;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _parent_str_onchange;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _parent_str.onChange -= _parent_str_onchange;
-      }
-
-      private void _parent_str_onchange(object dp_obj) {
-         Log.assert(dp_obj != null);
-         // the file _in_csv_file_name is now populated.
-         // call py_func_wrapper_main and pass this file name as well
-         // as the output file name
-         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
-         // transform function, which returns the tranformed df. py_func_wrapper_main then
-         // writes it into the _out_csv_filename file.
-         // now this function should read _out_csv_filename into a dp
-         // and fire it
-         _compute_and_write();
-         //ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
-         ViewStream rs = new CSVReaderStream(_out_csv_filename);
-         IBlueRowCollection nrows = null;
-         rs.onChange += (dp_obj) => {
-            nrows = (dp_obj as DataPacket).newRows;
-         };
-         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      private void _compute_and_write() {
-         File.WriteAllText("user_code.py", _transform_function);
-         // wrapper function
-         string py_func_wrapper_main = @"
-import sys
-from user_code import transform
-import pandas as pd
-
-if __name__ == '__main__':
-   input_file_name = sys.argv[1]
-   output_file_name = sys.argv[2]
-# 1. load input_file_name into df
-# 2. do some test calc
-# 3. write the output of 2 into output_file_name
-   df = pd.read_csv(input_file_name)   
-   df_out = transform(df)
-   df_out.to_csv(output_file_name, index=False)";
-
-
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-      }
-
-      #region Unused test code
-      private void _simple_test() {
-
-         // actual function that needs to be called ////////////////////////
-         string py_real_function = @"
-def test(x, out_path):
-   rv = x * 10
-   with open(out_path, 'w') as f:
-      f.write(str(rv))
-   return rv";
-         // the function is in a string object.
-         File.WriteAllText("user_code.py", py_real_function);
-
-
-         // runner app (top level main function) //////////////////////////////
-         string py_func_wrapper_main = @"
-import sys
-from user_code import test
-
-if __name__ == '__main__':
-   arg = int(sys.argv[1])
-   out_path = sys.argv[2]
-   rv = test(arg, out_path)
-   rv = rv + 12
-   with open(out_path, 'w') as f:to \
-      f.write(str(rv))";
-   
-         File.WriteAllText("runner.py", py_func_wrapper_main);
-
-         string outputPath = "py_out.txt";
-         ProcessStartInfo psi = new ProcessStartInfo {
-            FileName = "python",
-            Arguments = $"runner.py 4 {outputPath}",
-            RedirectStandardOutput = false,
-            UseShellExecute = false
-         };
-         var proc = Process.Start(psi);
-         proc.WaitForExit();
-
-         // Now read the output written by the function
-         string result = File.ReadAllText(outputPath);
-         Console.WriteLine(result); // Should print "6"
-
-      }
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SchemaGuesserStream.cs

@@ -1,30 +0,0 @@
-using BCG;
-
-namespace bts.BlueOrb {
-   public class SchemaGuesserStream : ViewStream {
-
-      private IBlueRowCollection _rows = null;
-      private ViewStream _vs;
-      public SchemaGuesserStream(ViewStream vs) {
-         _vs = vs;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vs.onChange += _on_data;
-      }
-      protected override void myOnLastSubscriber() {
-         _vs.onChange -= _on_data;
-      }
-
-      private void _on_data(object obj) {
-         DataPacket dp = obj as DataPacket;
-         if (_rows == null) {
-            _rows = BlueDataUtility.get_typed_table(dp.newRows);
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SingleRowMathStream.cs

@@ -1,67 +0,0 @@
-using Bts.ArithmeticExpressions;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using BCG;
-
-namespace bts.BlueOrb {
-
-   [DataStreamUserRespresentation("SingleRowMathStream")]
-   public class MathStream : ViewStream, IStreamMapper {
-      private IArithmeticProgram myExp;
-      private BooleanExpressionFilter _bool;
-      List<string> myExtraColumns = new List<string>();
-      BlueTableSchema mySchema;
-
-
-      public MathStream(ViewStream vs, string mathExpression)
-         : this(vs, mathExpression, string.Empty) {
-      }
-
-      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
-         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
-            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
-         myExtraColumns = myExp.OutputIdentifiers.ToList();
-         if (!string.IsNullOrEmpty(boolean_expression))
-            _bool = new BooleanExpressionFilter(boolean_expression);
-         mySetRealSource(new MapStream(vs, this));
-      }
-
-      #region IStreamMapper Members
-
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
-         if (mySchema == null) {
-            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
-            foreach (ColumnInfo ci in inputSchema.columnInfo)
-               colIdx[ci.colName] = ci;
-
-            foreach (string ncn in myExtraColumns) {
-               ColumnInfo newC;
-               if (colIdx.ContainsKey(ncn))
-                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
-               else
-                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
-               colIdx[ncn] = newC;
-            }
-            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
-         }
-         return mySchema;
-      }
-
-      #endregion
-
-      #region IDynamicStreamMapper Members
-
-      public IBlueRow map(IBlueRow row) {
-         MutableBlueRow result = new MutableBlueRow(mySchema, row);
-         if (_bool == null || _bool.filter(row)) {
-            myExp.Evaluate(result.AsDictionary());
-         }
-         return result;
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs

@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-
-   [DataStreamUserRespresentation("SortStream")]
-   public class SortStream: ViewStream {
-      public SortStream (ViewStream parent_stream, string sort_expr) {
-         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs

@@ -1,16 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Text;
-   using System.Threading.Tasks;
-   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
-   public class TimeBasedRollingFilterStream : ViewStream {
-      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
-         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/TopStream.cs

@@ -1,11 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("TopStream")]
-   public class TopStream : ViewStream { 
-      public TopStream(ViewStream vs, int n_rows, string sort_col) {
-         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -1,17 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-   using System.Collections.Generic;
-
-   using mbts;
-   using BCG;
-
-   // [DataStreamUserRespresentation("UnionStream")]
-   [DataStreamUserRespresentation("AppendStream")]
-   public class UnionStream : ViewStream {
-      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
-         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -1,42 +1,25 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
-    <PackageReference Include="IronPython" Version="3.4.2" />
-    <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
-    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
-    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
-    <PackageReference Include="Parquet.Net" Version="5.0.1" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+    <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
+    <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
+    <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
-  <Choose>
-    <When Condition="'$(UseProjectRefs)' == 'true'">
-      <ItemGroup>
-        <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-        <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
-        <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
-        <ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
-        <ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
-        <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
-      </ItemGroup>
-    </When>
-    <Otherwise>
-      <ItemGroup>
-        <PackageReference Include="ArithmeticExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="BaseUtilities" Version="$(SparkVersion)" />
-        <PackageReference Include="BooleanExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.ColumnExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="Bts.ColumnSortExpressions" Version="$(SparkVersion)" />
-        <PackageReference Include="DataStreamExpressions" Version="$(SparkVersion)" />
-      </ItemGroup>
-    </Otherwise>
-  </Choose>
+  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
+    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
+  </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,217 +2,25 @@ namespace Bts.DataStream;
 
 using BCG;
 
-using bts.api.core;
-
-using Bts.DataStreamCore;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Logging;
-
 using Spark.Core.Disposables;
-using Spark.Core.Threading;
 
 using System;
-using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
-   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
-      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
-   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> observer) {
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
          void OnChange(object? newValue) =>
-            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
-               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
-
-   // public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
-   //    source.Select(dp => {
-   //       dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-   //       return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
-
-   //       IBlueRowCollection<T> CastRows(ICollection rows) =>
-   //          new BlueTableSelectorWrapper<T>(
-   //             rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
-   //    });
-
-   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
-      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
-      source.AsTables<T, int>(valueSelector, r => r.Id());
-
-   public static IObservable<TableUpdate<T, int>> AsTables<T>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
-      source.AsTables<T, int>(valueSelector, keySelector);
-
-   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
-      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
-      rowKeySelector ??= r => keySelector(valueSelector(r));
-      return source.Scan(TableUpdate<T, TKey>.Empty(keySelector), (res, dp) => {
-         var bd = res.ToBuilder();
-         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
-         foreach (IBlueRow r in addedRows) {
-            bd.AddOrUpdate(valueSelector(r));
-         }
-         foreach (IBlueRow r in deletedRows) {
-            bd.Remove(rowKeySelector(r));
-         }
-         return bd.ToImmutable();
-      });
-   }
-
-   public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
-      source.Scan(
-         (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
-         (res, table) => {
-            var resTable = res.Table;
-            if (resTable is null) {
-               if (table.Index.Count == 0) {
-                  return (null, null);
-               }
-               resTable = new(table.Index.Values.First().schema);
-            }
-            var bd = resTable.ToDataPacketBuilder();
-            foreach (var r in table.RemovedItems) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.AddedItems) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
-         .Select(t => t.Dp!);
-
-   public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.Scan(
-         (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
-         (res, table) => {
-            var bd = res.Table.ToDataPacketBuilder();
-            foreach (var r in table.RemovedItems) {
-               bd.Remove(r.Id());
-            }
-            foreach (var r in table.AddedItems) {
-               bd.AddOrUpdate(r);
-            }
-            var resDp = bd.ToDataPacket(out var newRows);
-            return (newRows, resDp);
-         })
-         .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
-         .Select(t => t.Dp);
-
-   public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
-      return Observable.Create<IDataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<IDataPacket> o) {
-         var adapter = new BlueTableImmutableAdapter<T>(schema);
-         IDataPacket initDataPacket = new DataPacket(schema);
-         return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
-
-         (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
-            (BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
-            var bd = state.Adapter.ToDataPacketBuilder();
-            bd.Update(tableUpdate);
-            var newRows = bd.ToImmutable(out var dp);
-            return (newRows, dp);
-         }
-      }
-   }
-
-   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
-      source.AsDataPackets(schema).AsViewStream();
-
-   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
-      var indices = ResolveIndices(schema, new[] { name });
-      return indices[0];
-   }
-
-   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
-      var indices = ResolveIndices(schema, new[] { name1, name2 });
-      return (indices[0], indices[1]);
-   }
-
-   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
-      return (indices[0], indices[1], indices[2]);
-   }
-
-   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
-      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
-      return (indices[0], indices[1], indices[2], indices[3]);
-   }
-
-   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
-      var columns = schema.columnInfo;
-      var indices = new int[columnNames.Length];
-      int nResolved = 0;
-      for (int i = 0; i < columnNames.Length; i++) {
-         int origNResolved = nResolved;
-         for (int j = 0; j < columns.Length; j++) {
-            if (columns[j].colName == columnNames[i]) {
-               indices[i] = j;
-               nResolved++;
-               break;
-            }
-         }
-         if (nResolved == origNResolved) {
-            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
-         }
-      }
-      return indices;
-   }
-
-   public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
-      Observable.Create<IDataPacket>(o => {
-         var inputGate = new AtomicGate();
-         var outputGate = new AtomicGate();
-         BlueTableImmutable? lastTable = null;
-         BlueTableImmutable.DataPacketBuilder? bd = null;
-         IDataPacket? pendingDpOut = null;
-         return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
-
-         void OnNext(IDataPacket value) {
-            using (inputGate.BeginTransaction()) {
-               lastTable ??= new(value.newRows.schema);
-               bd ??= lastTable.ToDataPacketBuilder();
-               if (pendingDpOut is null) {
-                  // The last outgoing datapacket was just fired. Don't merge this update with the
-                  // previous one (otherwise its adds/removes would be fired twice).
-                  bd.Reset();
-               }
-               foreach (IBlueRow r in value.deletedRows) {
-                  bd.Remove(r.Id());
-               }
-               foreach (IBlueRow r in value.addedRows) {
-                  bd.AddOrUpdate(r);
-               }
-               pendingDpOut = bd.ToDataPacket(out lastTable);
-            }
-         }
-
-         void DoWork() {
-            IDataPacket? dpOut = null;
-            if (outputGate.TryEnter()) {
-               using var exit = Disposable.Create(outputGate.Exit);
-               (dpOut, pendingDpOut) = (pendingDpOut, dpOut);
-               if (dpOut is not null) {
-                  o.OnNext(dpOut);
-               }
-            }
-         }
-      });
-
-
 }

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -1,35 +0,0 @@
-namespace bts.BlueOrb {
-   using System;
-   using BCG;
-   using System.Collections.Generic;
-   using Bts.DataStreamExpressions;
-   using System.Xml;
-
-   // All this does is take in a dsl expression, parse it and fire the data that come out of running it.
-   // So for example, if an expresison is passed 
-   [DataStreamUserRespresentation("dsl")]
-   public class DSLStream : ViewStream {
-      private readonly string _dsl;
-      public DSLStream(string dsl) {
-         Log.Gen.info($"DSLStream.cons dsl: {dsl}");
-         // Remove all colons
-         string sanitizedDsl = dsl.Replace(":", "COLON");
-         _dsl = dsl;
-         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
-         mySetRealSource(rs);
-      }
-
-      static ViewStream? build_stream(string stream_expression) {
-         try {
-            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-            return rv;
-
-         } catch (Exception ex) {
-            //send to blueorb
-            Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
-            return null;
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -1,13 +1,14 @@
 namespace Bts.DataStream;
 
 using BCG;
+using BCG.Extensions;
 using BCG.Utilities.General;
 
 using bts;
+using bts.collections;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
 using Spark.Core.Disposables;
 
 using System;

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,21 +4,14 @@ using BCG;
 
 using bts;
 
-using Microsoft.Extensions.Logging;
-
 using System;
 
-public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDisposable {
-   readonly IObservable<IDataPacket> source;
-   readonly string? name;
-   readonly ILogger? logger;
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<IDataPacket> source, string? name = null, ILogger? logger = null) {
+   public ObservableViewStream(IObservable<DataPacket> source) =>
       this.source = source;
-      this.name = name;
-      this.logger = logger;
-   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -30,13 +23,7 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
-
-      void OnError(Exception exception) =>
-         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
-
-      void OnCompleted() =>
-         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
+      subscription = source.Subscribe(myFireEvent);
    }
 
    protected override void myOnLastSubscriber() {
@@ -45,6 +32,6 @@ public class ObservableViewStream : ViewStream, IObservable<IDataPacket>, IDispo
       subscription = null;
    }
 
-   IDisposable IObservable<IDataPacket>.Subscribe(IObserver<IDataPacket> observer) =>
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
       source.Subscribe(observer);
 }

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -1,25 +1,14 @@
-using bts;
-
-using Bts.DataStream;
-using Bts.DataStreamCore;
-using Bts.DataStreamExpressions;
-using Bts.PubSub;
-
-using Microsoft.Extensions.Hosting;
-
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
+
+using Bts.DataStreamExpressions;
+using BCG.Utilities.General;
+using bts;
 
 namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IHostedService {
+   public class RemoteQueryManager : IWorker {
       public class StreamPubUnit {
          public string name;
          public string stream_expression;
@@ -50,45 +39,23 @@ namespace BCG {
       }
       private IDictionary<string, StreamPubUnit>
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      readonly IPubTableBroker _broker;
-      readonly IManagedSubscriber _sub;
-      readonly object syncRoot = new();
-      IDisposable? cloudStreamsSub;
-      IDisposable? remoteRequestsSub;
-      bool isDisposed;
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
 
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
-         _sub = managedSubscriber;
-         _broker = pubTableBroker;
+      public RemoteQueryManager() {
+         _stream.onChange += _on_str;
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
-
-      public Task StartAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            Log.assert(!isDisposed);
-            Log.isnull(cloudStreamsSub);
-            Log.isnull(remoteRequestsSub);
-            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
-            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
-               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-            });
+      private void _on_str(object obj)
+      {
+         if (obj is not DataPacket dp) {
+            return;
          }
-         return Task.CompletedTask;
-      }
-
-      public Task StopAsync(CancellationToken cancellationToken) {
-         lock (syncRoot) {
-            if (!isDisposed) {
-               isDisposed = true;
-               Disposable.dispose(ref cloudStreamsSub);
-               Disposable.dispose(ref remoteRequestsSub);
-            }
-         }
-         return Task.CompletedTask;
-      }
-
-      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -105,10 +72,7 @@ namespace BCG {
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
          if (persist) {
-            string tempDir = Path.GetTempPath();
-        Directory.CreateDirectory(tempDir); // make sure it exists
-
-        stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
          StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
@@ -118,5 +82,9 @@ namespace BCG {
       private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
+
+      public void doWork() {
+
+      }
    }
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -9,7 +9,7 @@ namespace BCG {
    [DataStreamUserRespresentation("remoteQuery")]
    public class RemoteQueryStream : ViewStream {
       private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = PublisherCore.Create();
+      private PublisherCore _publisher_core = new PublisherCore();
       private ViewStream _source_str;
 
       public RemoteQueryStream(params string[] lines) {

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -2,6 +2,9 @@ namespace bts {
    using System.IO;
    using BCG;
 
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
    public class RemoteRequest {
       public readonly string stream_expression;
       public readonly string published_table_name;
@@ -25,4 +28,4 @@ namespace bts {
 
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,46 +1,33 @@
-using Spark.Core.Collections;
-
 using System;
-using System.Collections.Generic;
-using System.Linq;
 
-namespace BCG {   
+namespace BCG {
    public class AddColumnsStream : ViewStream {
-      public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
          string[] toks = exp.Split(':');
-         string col_name = toks[0].Trim();
-         string col_value = toks[1].Trim();
+         string col_name = toks[0];
+         string col_value = toks[1];
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return (col_name, tup.Item1, tup.Item2);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
       }
 
-      public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
-         exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
-
-      readonly List<(string Name, Type Type, object Value)> _new_col_map;
-      IBlueTableSchema? _out_schema;
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col_map = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, _get_schema, _map);
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
          mySetRealSource(rs);
       }
 
       private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null) {
-            _out_schema = input_schema;
-            foreach (var m in _new_col_map) {
-               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
-            }
-         }
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
          return _out_schema;
       }
 
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         foreach (var m in _new_col_map) {
-            rv.setValue(m.Name, m.Value);
-         }
+         rv.setValue(_new_col.Item1, _new_col.Item3);
          return rv;
       }
    }

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -1,10 +1,9 @@
-using mbts;
-
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
 
 namespace BCG
 {
@@ -34,7 +33,7 @@ namespace BCG
 
          private void _stream_onChange(object newValue)
          {
-            var dp = (IDataPacket)newValue;
+            DataPacket dp = newValue as DataPacket;
             if (dp != null && dp.newRows.Count > 0)
             {
                _stream.onChange -= _stream_onChange;
@@ -43,8 +42,8 @@ namespace BCG
          }
       }
 
-      public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
-      public delegate string GetColumnPrefix(object?[] transposing_col_values);
+      public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
+      public delegate string GetColumnPrefix(object[] transposing_col_values);
 
       public delegate string GetColumnSuffix(string col_name);
       private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -52,31 +51,33 @@ namespace BCG
       private IncrementValues _increment_values;
       private GetColumnPrefix _get_column_prefix;
 
-      IBlueTableSchema? _schema;
+      private IBlueTableSchema _schema;
 
       private string[] _group_by_cols;
       private Pair<string, Type>[] _cols_per_transposed;
       private string[] _transposing_cols;
 
-      int[]? _transposing_col_indices;
+      private int[] _transposing_col_indices;
 
-      Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
+      private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
          : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
                 get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
       { }
 
       public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
-         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
+         IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
       {
          if (get_column_suffix_del != null)
             _get_column_suffix = get_column_suffix_del;
          if (transposed_col_value_comparer != null)
          {
-            _transpsed_col_value_comparer_wrapper = new((p1, p2) =>
-               transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
+            _transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
+               {
+                  return transposed_col_value_comparer(p1.second, p2.second);
+               });
          }
          _group_by_cols = group_by_cols.Split(' ');
          _transposing_cols = transposed_cols_group.Split(' ');
@@ -105,7 +106,7 @@ namespace BCG
             col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -129,14 +130,19 @@ namespace BCG
 
       public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
       {
-         var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
-         var group_by_cols_value_dict = new Dictionary<int, object?[]>();
+         Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
+         Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
          BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
          Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
          string table_name = string.Empty;
          if (grouped_rows.Count > 0)
          {
-            var schema = grouped_rows.Values.First()[0].schema;
+            IBlueTableSchema schema = null;
+            foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
+            {
+               schema = kvp.Value[0].schema;
+               break;
+            }
             table_name = schema.tableName;
             _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
             int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -149,22 +155,23 @@ namespace BCG
          else
             return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); 
          Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
-         var tc_values = new List<Pair<int, object?[]>>();
+         List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
          foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
          {
             //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values 
-            var incrementing_values = new Dictionary<int, object?[]>();
+            Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
             foreach (IBlueRow r in kvp.Value)
             {
                int hash = hasher.get_hash(r, _transposing_col_indices);
-               if (!incrementing_values.TryGetValue(hash, out var values))
+               object[] values;
+               if (!incrementing_values.TryGetValue(hash, out values))
                {
                   incrementing_values.Add(hash, _get_per_transposing_col_values(r));
                   string prefix = _get_column_prefix(_get_transposing_col_values(r));
                   if (!tc_prefixes.ContainsKey(hash))
                   {
                      tc_prefixes.Add(hash, prefix);
-                     tc_values.Add(new(hash, _get_transposing_col_values(r)));
+                     tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
                   }
                }
                else
@@ -187,18 +194,18 @@ namespace BCG
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);
          int id = 1;
-         foreach (var kvp in all_tc_infos_and_values)
+         foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
          {
             MutableBlueRow mbr = new MutableBlueRow(_schema);
             mbr.setValue("id", id++);
             for (int i = 0; i < _group_by_cols.Length; i++)
                mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
-            foreach (var tc_cols in kvp.Value)
+            foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
             {
                string prefix = tc_prefixes[tc_cols.Key];
                for (int i = 0; i < _cols_per_transposed.Length; i++)
                {
-                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
+                  string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
                   if(mbr.schema.columnNames.Contains(column))
                      mbr.setValue(column, tc_cols.Value[i]);
                }
@@ -210,25 +217,28 @@ namespace BCG
 
       #endregion
 
-      object?[] _get_transposing_col_values(IBlueRow row) {
-         var values = new object?[_transposing_cols.Length];
+      private object[] _get_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_transposing_cols.Length];
          for (int i = 0; i < _transposing_cols.Length; i++)
             values[i] = row[_transposing_cols[i]];
          return values;
       }
 
-      object?[] _get_per_transposing_col_values(IBlueRow row) {
-         var values = new object?[_cols_per_transposed.Length];
+      private object[] _get_per_transposing_col_values(IBlueRow row)
+      {
+         object[] values = new object[_cols_per_transposed.Length];
          for (int i = 0; i < _cols_per_transposed.Length; i++)
-            values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
+            values[i] = row[_cols_per_transposed[i].first];
          return values;
       }
 
-      object?[] _get_group_by_cols_values(IBlueRow row) {
-         var values = new object?[_group_by_cols.Length];
+      private object[] _get_group_by_cols_values(IBlueRow row)
+      {
+         object[] values = new object[_group_by_cols.Length];
          for (int i = 0; i < values.Length; i++)
             values[i] = row[_group_by_cols[i]];
          return values;
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -1,47 +1,52 @@
 using System;
 
-namespace bts.BlueOrb {
-   using BCG;
-   [DataStreamUserRespresentation("CountStream")]
-   public class CountStream : ViewStream {
+namespace BCG
+{
+   [DataStreamUserRespresentation ("count")]
+	public class CountStream : ViewStream
+	{
       private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema("count",
-         new ColumnInfo("id", 0, typeof(int)),
-         new ColumnInfo("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("count", 1, typeof(int)));
       private int myId = 0;
       private ViewStream myVS;
 
-      public CountStream(ViewStream vs) {
+      public CountStream (ViewStream vs)
+      {
          myVS = vs;
       }
 
-      public CountStream(string tableName)
-         : this(new TableStream(tableName)) { }
+      public CountStream (string tableName)
+         : this (new TableStream (tableName))
+      {}
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber () {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber () {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 
-      private void myHandleChanges(object newValue) {
-         myFireEvent(
+      private void myHandleChanges(object newValue)
+      {
+         myFireEvent (
             myLastFiredDataPacket =
                myLastFiredDataPacket == null ?
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
-                  new BlueTable(mySchema))
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new BlueTable (mySchema))
                   :
-                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
                   myLastFiredDataPacket.newRows)
             );
       }
 
-      private IBlueRowCollection myGetRows(object dp, int id) {
-         BlueTable retVal = new BlueTable(mySchema);
+      private IBlueRowCollection myGetRows (object dp, int id)
+      {
+         BlueTable retVal = new BlueTable (mySchema);
          retVal.addRow
-            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -1,19 +1,22 @@
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 namespace BCG
 {
    public class DerivativeStream : ViewStream 
    {
-      IBlueTableSchema? mySchema;
+
+      private IBlueTableSchema mySchema;
       private string myDerivandColumn;
       private string myOutputColumn;
       private string myKeyColumn;
       private DateTime myLastPacketArrival = DateTime.MinValue;      
       private ViewStream myDataSource;
-      Dictionary<string, double>? myPreviousValues;
+      
+      private IDictionary<string, double> myPreviousValues;
+
 
       public DerivativeStream(ViewStream input, string keyColumn,  string columnToDifferentiate, string outputColName)
       {
@@ -36,9 +39,9 @@ namespace BCG
 
       void myDataSource_onChange(object newValue)
       {
-         var dp = (IDataPacket)newValue;
+         DataPacket dp = (DataPacket)newValue;
          if (myLastPacketArrival == DateTime.MinValue)
-            mySchema = myBuildOutputSchema(dp.Schema);
+            mySchema = myBuildOutputSchema(dp.schema);
          else
          {
             DateTime now = DateTime.Now;
@@ -52,7 +55,7 @@ namespace BCG
             else
             {
                IBlueRowCollection deleted, added;
-               BlueDataUtility.computeDifferences(thisVersion, ((IDataPacket)myLastValue).newRows, out deleted, out added);
+               BlueDataUtility.computeDifferences(thisVersion, ((DataPacket)myLastValue).newRows, out deleted, out added);
                myFireEvent(new DataPacket(thisVersion, added, deleted));  
             }
          }
@@ -61,13 +64,12 @@ namespace BCG
 
       protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
       {
-         ExceptionAssert.NotNull(mySchema, myPreviousValues);
          object[] outValues = new object[mySchema.columnInfo.Length];
          r.Values.CopyTo(outValues, 0);
 
-         var derivand = r[myDerivandColumn];
-         var key = getRowKey(r);
-         if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
+         object derivand = r[myDerivandColumn];
+         double lastValue = default(double);
+         if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
             outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
          else
             outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -82,19 +84,16 @@ namespace BCG
          cols[cols.Length - 1] = new ColumnInfo(myOutputColumn, cols.Length - 1, typeof(double));
          return new BlueTableSchema("derviative of " + inSchema.tableName, cols);
       }
-
-      protected void myStoreNewPacketInfo(IDataPacket dp) {
+      protected void myStoreNewPacketInfo(DataPacket dp)
+      {
          myPreviousValues = new Dictionary<string, double>();
          myLastPacketArrival = DateTime.Now;
          foreach (IBlueRow row in dp.newRows)
          {
-            var derivand = row[myDerivandColumn];
+            object derivand = row[myDerivandColumn];
             if (derivand != null && derivand != DBNull.Value)
-               myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
+               myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
          }
       }
-
-      string getRowKey(IBlueRow row) =>
-         row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
    }
 }

src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs

@@ -1,9 +1,7 @@
-using mbts;
-
-using Spark.Core.Collections;
-
-using System;
+using System;
+using System.Collections;
 using System.Collections.Generic;
+using mbts;
 
 namespace BCG
 {
@@ -14,9 +12,9 @@ namespace BCG
    {
       ViewStream myLStream, myRStream;
 
-      DataPacket? myLastR;
-      DataPacket? myLastL;
-      BlueTable? myLastFired;
+      DataPacket myLastR = null, myLastL = null;
+
+      BlueTable myLastFired = null;
 
       double myThreshold = .00005;
 
@@ -26,14 +24,14 @@ namespace BCG
       public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : 
          this(null, v1, v2, thresh, ColumnsToMatch) { }
 
-      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
       {
          myThreshold = thresh;
          myLStream = v1;
          myRStream = v2;
          myColumnsToMatch = columnsToMatch;
       }
-      public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
+      public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
          : this(db,v1,v2,.00005,columnsToMatch)
       {
       }
@@ -52,22 +50,45 @@ namespace BCG
          myRStream.onChange -= myRChange;
       }
 
+      struct ListComparer : IEqualityComparer<List<object>>
+      {
+         public bool Equals(List<object> x, List<object> y)
+         {
+            if (x.Count != y.Count)
+               return false;
+
+            for (int i = 0; i < x.Count; ++i)
+               if (!x[i].Equals(y[i]))
+                  return false;
+            return true;
+         }
+
+         public int GetHashCode(List<object> obj)
+         {
+            int ret = 0;
+            for(int i = 0; i < obj.Count; ++i)
+               ret ^= obj[i].GetHashCode();
+            return ret;
+         }
+      }
+
       void myHandleCallbacks()
       {
-         DataPacket? packet;
+         DataPacket packet = null;
+
          if (myLastL == null || myLastR == null)
          {
             packet = myLastL == null ? myLastR : myLastL;
-            if (packet is not null) {
+            if (packet != null)
                myFireEvent(packet);
-               myLastFired = new BlueTable(packet.newRows);
-            }
+
+            myLastFired = new BlueTable(packet.newRows);
             return;
          }
 
          BlueTable table = new BlueTable(myLastL.newRows.schema);
 
-         var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
+         Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
 
          foreach(BlueRow row in myLastL.newRows)
          {
@@ -97,7 +118,11 @@ namespace BCG
                }
                catch (Exception) { }
             }
-            rowPairs.TryGetValue(hash, out var pair);
+
+            Pair<BlueRow, BlueRow> pair;
+            rowPairs.TryGetValue(hash, out pair);
+
+
             if (pair == null)
                pair = new Pair<BlueRow, BlueRow>(null, row);
             else
@@ -113,7 +138,8 @@ namespace BCG
 
             if (pair.first == null || pair.second == null)
             {
-               var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
+               BlueRow row = pair.first == null ? pair.second : pair.first;
+
                Dictionary<string, object> newRow = new Dictionary<string, object>();
                foreach (ColumnInfo info in row.schema.columnInfo)
                {
@@ -185,4 +211,4 @@ namespace BCG
 
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -14,8 +14,10 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
          : base (colMapExpression, null, columnNames) {}
 
-      public override IBlueRow? evaluate(IDataPacket packet) {
-         myOutputTable ??= new GroupByOutputTable(packet.Schema);
+
+      public override IBlueRow evaluate (DataPacket packet)
+      {
+         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
          string hashString = GroupByStream.getRepString (packet, myGroupByCols);
          if (packet.newRows.Count == 0)
             //if there are no rows for this group we remove the group from the output table

src/DataStreamSuite/DataStream/Primitives/File/FileViewStream.cs

@@ -5,7 +5,7 @@ namespace BCG
 {
 	public class FileViewStream : ViewStream
 	{
-      DataPacket? myDp;
+      private DataPacket myDp;
 		public FileViewStream (string dataFileName, 
          string schemaFileName)
          : this (dataFileName, schemaFileName, Int32.MaxValue)

src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs

@@ -1,7 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
-
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 namespace BCG
 {
     public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -25,14 +26,20 @@ namespace BCG
             myReference = myTodaysDate.AddDays(-1 * myNDays);                     
         }       
         
-        public event FilterChangeDelegate? onFilterChange;       
+        public event FilterChangeDelegate onFilterChange;       
         
-        public bool filter(IBlueRow r) =>
-           r[myDateColumn] switch {
-              DBNull or null => true,
-              DateTime t => t >= myReference,
-              _ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
-           };
+        public bool filter(IBlueRow r)
+        {
+            if (DBNull.Value.Equals(r[myDateColumn]))
+            {
+                //return false;
+                return true; //it can be considered as our assumption  whenever datecolumn is db null we return true 
+            }
+            DateTime t = (DateTime)r[myDateColumn];                   
+            bool returnVallue =  t < myReference ? false : true;
+            return returnVallue;
+        }
+              
 
         public void doWork()
         {
@@ -40,7 +47,7 @@ namespace BCG
             {
                 myTodaysDate = DateTime.Today;
                 myReference = myTodaysDate.AddDays(-1 * myNDays);
-                onFilterChange?.Invoke();
+                onFilterChange();
             }         
         }      
     }

src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs

@@ -1,6 +1,8 @@
-using BCG.Utilities.General;
-
-using System;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG.Utilities.General;
 
 namespace BCG
 {
@@ -24,7 +26,7 @@ namespace BCG
 
       #region IDynamicStreamFilter Members
 
-      public event FilterChangeDelegate? onFilterChange;
+      public event FilterChangeDelegate onFilterChange;
 
       #endregion
 
@@ -32,10 +34,7 @@ namespace BCG
 
       public bool filter(IBlueRow r)
       {
-         DateTime rowDate = r[myDateColName] switch {
-            DBNull or null => DateTime.Now.Date,
-            object obj => (DateTime)obj,
-         };
+         DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
          DateTime today = DateTime.Now.Date;
          return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
       }
@@ -49,7 +48,7 @@ namespace BCG
          if (DateTime.Now.Date != myLastCheckedTime.Date)
          {
             myLastCheckedTime = DateTime.Now;
-            onFilterChange?.Invoke();
+            onFilterChange();
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,109 +1,93 @@
-using bts;
+using Bts.DataStream;
 
-using Bts.DataStream.Primitives.Filter;
-using Bts.DataStreamCore;
-
-using Spark.Core.Diagnostics;
-using Spark.Core.Disposables;
+using mbts.config;
 
 using System;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
 using System.Text.RegularExpressions;
 
-namespace BCG.StreamWorld.Addition {
-   [DataStreamUserRespresentation(false)]
-   public class FilterStream : ViewStream {
+namespace BCG.StreamWorld.Addition
+{
+   [DataStreamUserRespresentation (false)]
+   public class FilterStream : ViewStream
+   {
       private ViewStream myBaseStream;
-      BlueTable? myBaseTable;
-      BlueTable? myStateTable;
+      private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
-         : this(vs, "1=1") { }
+         : this(vs, "1=1")
+      { }
 
       public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp)) { }
+         : this(vs, new BooleanExpressionFilter(filterExp))
+      { }
 
-      public FilterStream(ViewStream vs, IStreamFilter filter) {
+      public FilterStream(ViewStream vs, IStreamFilter filter)
+      {
          myBaseStream = vs;
          myFilter = filter;
-         vs.ErrorChanged += OnError;
-         if (filter is IDynamicStreamFilter dfilter)
-            dfilter.onFilterChange += myHandleFilterChange;
+         if (filter is IDynamicStreamFilter)
+            (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
       }
 
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
-         myBaseTable?.clear();
-         myStateTable?.clear();
+         myBaseTable.clear();
+         myStateTable.clear();
       }
 
-      bool filterRow(IBlueRow r) {
-         try {
-            return myFilter.filter(r);
-         } catch (Exception ex) {
-            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+      private void myHandleChanges(object newValue)
+      {
+         DataPacket dp = newValue as DataPacket;
+         if (myStateTable == null)
+         {
+            bts.Log.assert(myBaseTable == null);
+            myStateTable = new BlueTable(dp.schema);
+            myBaseTable = new BlueTable(dp.schema);
          }
-         return false;
-      }
-
-      private void myHandleChanges(object newValue) {
-         try {
-            var dp = (IDataPacket)newValue;
-            var schema = dp.Schema;
-            init(schema);
-            var dels = new BlueTable(schema);
-            var adds = new BlueTable(schema);
-            foreach (IBlueRow r in dp.deletedRows) {
-               myBaseTable.deleteRow(r);
-               if (myStateTable.containsID(r.Id())) {
-                  dels.addRow(r);
-                  myStateTable.deleteRow(r);
-               }
+         BlueTable dels = new BlueTable(dp.schema),
+                     adds = new BlueTable(dp.schema);
+         foreach (IBlueRow r in dp.deletedRows)
+         {
+            myBaseTable.deleteRow(r);
+            if (myStateTable.containsID((int)r["id"]))
+            {
+               dels.addRow(r);
+               myStateTable.deleteRow(r);
             }
-            foreach (IBlueRow r in dp.addedRows) {
-               myBaseTable.addRow(r);
-               if (filterRow(r)) {
-                  adds.addRow(r);
-                  myStateTable.addRow(r);
-               }
-            }
-            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-               myFireEvent(new DataPacket(myStateTable, adds, dels));
-         } catch (Exception ex) {
-            OnError(ex);
          }
+         foreach (IBlueRow r in dp.addedRows)
+         {
+            myBaseTable.addRow(r);
+            if (myFilter.filter(r))
+            {
+               adds.addRow(r);
+               myStateTable.addRow(r);
+            }
+         }
+         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
+            myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
-      public IStreamFilter filter {
-         set {
+      public IStreamFilter filter
+      {
+         set
+         {
             myFilter = value;
-            if (myBaseTable != null) {
-               ExceptionAssert.NotNull(myStateTable);
+            if (myBaseTable != null)
+            {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
-               // If the filter throws an exception, we can rethrow it as a StreamFilterException
-               // and include the 'currentRow' (for which the filter failed) to provide context.
-               IBlueRow? currentRow = null;
-               foreach (IBlueRow r in myBaseTable) {
-                  try {
-                     currentRow = r;
-                     if (filterRow(r)) {
-                        myStateTable.addRow(r);
-                     }
-                  } catch (Exception ex) {
-                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
-                  }
-               }
+               foreach (IBlueRow r in myBaseTable)
+                  if (myFilter.filter(r))
+                     myStateTable.addRow(r);
                IBlueRowCollection dels, adds;
                BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
                myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -115,17 +99,6 @@ namespace BCG.StreamWorld.Addition {
       //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
 
       private void myHandleFilterChange() { filter = myFilter; }
-
-      [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
-      void init(IBlueTableSchema schema) {
-         if (myStateTable is null) {
-            Log.isnull(myBaseTable);
-            myStateTable = new BlueTable(schema);
-            myBaseTable = new BlueTable(schema);
-         } else {
-            Log.notnull(myBaseTable);
-         }
-      }
    }
 }
 
@@ -149,24 +122,29 @@ namespace BCG
       }
 
       #region IStreamFilter Members
-      public bool filter(IBlueRow dr) =>
-         myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
-            DBNull or null => string.Empty,
-            object obj => obj.ToString() ?? string.Empty
-         }));
+      public bool filter(IBlueRow dr)
+      {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++)
+         {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
       #endregion
    }
 
    public class FilterStream : ViewStream
    {
-      BlueTable? myLastFiredRows;
-      IBlueRowCollection? myLastReceivedRows;
-      IStreamFilter? myFilter;
-      readonly ViewStream? myParentStream;
-      string? myFilterString;
-      readonly StreamWorld.Addition.FilterStream? myFilterStream;
-      readonly bool myUseNewFilter = false;
-      IDisposable? subscription;
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
+      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
 
       #region constructors
       public delegate bool OnFilter(IBlueRow r);
@@ -196,10 +174,9 @@ namespace BCG
          {
             myParentStream = vs;
             myFilter = filter;
-            myFilterString = filter.ToString() ?? string.Empty;
-            if (filter is IDynamicStreamFilter dfilter) {
-               dfilter.onFilterChange += myHandleFilterChange;
-            }
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
          }
       }
 
@@ -223,20 +200,18 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber() {
-         ExceptionAssert.IsNull(subscription);
-         subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
-      }
+      protected override void myOnFirstSubscriber()
+      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         Disposable.Dispose(ref subscription);
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
       }
 
       protected override void myFireEvent(object o)
       {
-         var dp = (IDataPacket)o;
+         DataPacket dp = o as DataPacket;
          if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
             base.myFireEvent(o);
       }
@@ -245,39 +220,34 @@ namespace BCG
       private void myHandleCallbacks(object newValue)
       {
          DataPacket retVal;
-         var arrivedPacket = (IDataPacket)newValue;
+         DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
-         try {
-            if (myLastFiredRows != null)
-            {
-               var addedRows = new BlueTable(myLastFiredRows.schema);
-               var deletedRows = new BlueTable(myLastFiredRows.schema);
-               retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
-               foreach (IBlueRow r in arrivedPacket.deletedRows)
-                  if (myLastFiredRows.containsID(r.Id()))
-                  {
-                     myLastFiredRows.deleteRow(r);
-                     deletedRows.addRow(r);
-                  }
-               foreach (IBlueRow r in arrivedPacket.addedRows)
-                  if (filterRow(r))
-                  {
-                     myLastFiredRows.addRow(r);
-                     addedRows.addRow(r);
-                  }
-            }
-            else
-            {
-               myLastFiredRows = new BlueTable(arrivedPacket.Schema);
-               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-               foreach (IBlueRow r in arrivedPacket.newRows)
-                  if (filterRow(r))
-                     myLastFiredRows.addRow(r);
-            }
-            myFireEvent(retVal);
-         } catch (Exception ex) {
-            OnError(ex);
+         if (myLastFiredRows != null)
+         {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"]))
+               {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r))
+               {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
          }
+         else
+         {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
       }
 
       public IStreamFilter filter
@@ -286,7 +256,7 @@ namespace BCG
          {
             if (myUseNewFilter)
             {
-               requireNewFilter().filter = filter;
+               myFilterStream.filter = filter;
             }
             else
             {
@@ -295,29 +265,25 @@ namespace BCG
                {
                    if (myLastReceivedRows != null)
                    {
-                     try {
                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
                        foreach (IBlueRow r in myLastReceivedRows)
-                           if (filterRow(r))
+                           if (myFilter.filter(r))
                                nRows.addRow(r);
                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
                           aRows = new BlueTable(nRows.schema);                      
                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                        myLastFiredRows = nRows;
                        myFireEvent(new DataPacket(nRows, aRows, dRows));
-                     } catch (Exception ex) {
-                        OnError(ex);
-                     }
-                  }
+                   }
                }
             }
          }
          get
          {
             if (myUseNewFilter)
-               return requireNewFilter().filter;
+               return myFilterStream.filter;
             else
-               return requireOldFilter();
+               return myFilter;
          }
       }
 
@@ -330,25 +296,7 @@ namespace BCG
 
       private void myHandleFilterChange()
       {
-         filter = requireOldFilter();
+         filter = myFilter;
       }
-
-      bool filterRow(IBlueRow r) {
-         try {
-            return requireOldFilter().filter(r);
-         } catch (Exception ex) {
-            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
-            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
-         }
-         return false;
-      }
-
-      [MemberNotNull(nameof(myFilterStream))]
-      StreamWorld.Addition.FilterStream requireNewFilter() =>
-         ExceptionAssert.MemberNotNull(myFilterStream);
-
-      [MemberNotNull(nameof(myFilter))]
-      IStreamFilter requireOldFilter() =>
-         ExceptionAssert.MemberNotNull(myFilter);
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -1,24 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-
-   public class NullFilterStream : ViewStream {
-      public NullFilterStream(ViewStream parent_str, string col_name)
-      : this(parent_str, col_name, true) { }
-      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
-         ViewStream rs = new FilterStream(parent_str,
-         r => {
-            bool is_dbnull = r.is_null(col_name);
-            if (is_dbnull) {
-               return false;
-            }
-            if (r.schema[col_name].type_code() == TypeCode.Double) {
-               return !Double.IsNaN(r.get<double>(col_name));
-            }
-            return true;
-         });
-         mySetRealSource(rs);
-      }
-   }
-}