rakurai excanges should automatically work on sim

Too long for Windows to clone externals (at least for the Windows Server
box running jenkins)

.dotnet-cli/.nuget/packages/microsoft.net.test.sdk/17.10.0/build/netcoreapp3.1/Microsoft.NET.Test.Sdk.Program.cs

@@ -1,4 +0,0 @@
-// <auto-generated> This file has been auto generated. </auto-generated>
-using System;
-[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
-class AutoGeneratedProgram {static void Main(string[] args){}}

.env

@@ -29,6 +29,9 @@
 #     tools/deploy.sh --help
 #
 
+### Host/Runtime ###
+TZ=Etc/UTC
+
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -38,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
+# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
+# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
+# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -45,6 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development

.gitignore

@@ -64,9 +64,3 @@ trace_data/
 *.lst
 .env.local
 .secrets
-.dotnet-cli/
-.dotnet-cli/
-.dotnet-cli /
-.dotnet/
-.codex_restore*.log
-src/cs/Bts.Spark.Core/.codexobj/

.vscode/launch.json

@@ -203,71 +203,9 @@
         "program": "PubSubServer.exe",
       },
       "cwd": "build/Debug/bin/net8.0",
-      // "env": {
-      //   "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      // },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DSPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DSPublisher",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
-      "windows": {
-        "program": "DSPublisher.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DSPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DSPublisher",
-      },
-      "windows": {
-        "program": "DSPublisher.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PersistenceServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PersistenceServer",
-      },
-      "windows": {
-        "program": "PersistenceServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PersistenceServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PersistenceServer",
-      },
-      "windows": {
-        "program": "PersistenceServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },

Spark.Api.Server.sln

@@ -51,6 +51,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{70E3DCF5-A3C3-45A0-8F18-B01B2D446532}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_command_bridge", "src\client\spark_command_bridge\spark_command_bridge.csproj", "{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -571,6 +575,30 @@ Global
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.ActiveCfg = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.Build.0 = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.ActiveCfg = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.Build.0 = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.ActiveCfg = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.Build.0 = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.Build.0 = Debug|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -592,6 +620,8 @@ Global
 		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{70E3DCF5-A3C3-45A0-8F18-B01B2D446532} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED} = {70E3DCF5-A3C3-45A0-8F18-B01B2D446532}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}

cmake/InitPythonModule.cmake

@@ -40,7 +40,13 @@ function(init_python_module)
   add_custom_command(
     OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
     DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMAND
+      ${CMAKE_COMMAND} -E env
+      "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}"
+      ${Python3_EXECUTABLE} -m pybind11_stubgen
+      --numpy-array-use-type-var
+      --ignore-invalid-expressions '.*'
+      ${_name}_module
     COMMENT "Generating py stubs"
   )
   add_custom_target(

compose.yaml

@@ -2,6 +2,7 @@ services:
   sparkd:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -9,24 +10,38 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+      - 8008:8008
+      - 8080:8080
+      - 9443:9443
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-shell:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+    build:
+      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: "do sleep 1; done"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -42,6 +57,7 @@ services:
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
       - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
@@ -50,26 +66,8 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -77,6 +75,36 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+      - 8008:8008
+      - 8080:8080
+      - 9443:9443
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+  sparkd-multicast:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -84,7 +112,6 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:
@@ -195,3 +222,6 @@ volumes:
     name: "md_publisher_22019"
   historic_md:
     name: "historic_md"
+  md_recording:
+    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
+    external: true

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y ca-certificates
 RUN apt-get update && apt-get install -y libssh2-1-dev
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-get update && apt-get install -y sudo
+RUN apt-get update && apt-get install -y tzdata
 
 # GCC
 ARG gcc_major=13
@@ -18,6 +19,13 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
   && apt-get install -y dotnet8
 
+RUN apt-get update && apt-get install -y libpq-dev
+RUN apt-get update && apt-get install -y libsodium-dev
+RUN apt-get update && apt-get install -y fish
+RUN apt-get update && apt-get install -y less
+RUN apt-get update && apt-get install -y vim
+
+
 RUN locale-gen en_US.UTF-8
 
 ARG docker_user="bts"

docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile

@@ -98,7 +98,7 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 RUN apt-get update && apt-get -y install libzstd-dev
 
 # Rakurai
-ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
+ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.12_amd64.deb"
 RUN --mount=type=secret,id=bitbucket_access_token \
   set -eux \
     && apt-get update \

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+  fish \
+  less \
+  vim
+
+USER bts
+WORKDIR /home/bts

spark_client.sln

@@ -1,433 +1,306 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
 		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
 		Debug|x86 = Debug|x86
-		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
-		Release|x64 = Release|x64
 		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
 		Release|x86 = Release|x86
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
+		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-   "dotnet.preferCSharpExtension": true
-}

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -15,18 +15,21 @@ static class ArgumentComparer {
       }
       var obj1 = value1.GetValue(env);
       var obj2 = value2.GetValue(env);
-      if (obj1 is null) {
-         if (obj2 is null) {
+      if (isNullLike(obj1)) {
+         if (isNullLike(obj2)) {
             return 0;
          }
          return -1;
       }
-      if (obj2 is null) {
+      if (isNullLike(obj2)) {
          return 1;
       }
-      return compareObjs(obj1, obj2);
+      return compareObjs(obj1!, obj2!);
    }
 
+   static bool isNullLike(object? value) =>
+      value is null || DBNull.Value.Equals(value);
+
    static int compareObjs(object obj1, object obj2) {
       try {
          var typ1 = obj1.GetType();

src/DataStreamSuite/DSPublisher/Configurations/.gitkeep

@@ -1 +0,0 @@
-

src/DataStreamSuite/DSPublisher/DSLWSClient/dsl_websocket_client.py

@@ -1,168 +0,0 @@
-"""
-Simple DSPublisher DSL websocket client.
-
-Example:
-   python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
-   python dsl_websocket_client.py --dsl-file query.dsl
-"""
-
-import argparse
-import asyncio
-import json
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-
-DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
-DEFAULT_URL = "ws://localhost:19093/ws/"
-
-
-def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
-   if dsl and dsl_file:
-      raise ValueError("Pass either --dsl or --dsl-file, not both.")
-
-   if dsl_file:
-      return Path(dsl_file).read_text(encoding="utf-8").strip()
-
-   if dsl:
-      return dsl.strip()
-
-   if DEFAULT_DSL.startswith("#"):
-      raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
-
-   return DEFAULT_DSL
-
-
-def load_config_path(config_path: str | None) -> Path | None:
-   if config_path:
-      return Path(config_path)
-
-   environment = os.environ.get("DOTNET_ENVIRONMENT")
-   config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
-   script_dir = Path(__file__).resolve().parent
-   candidate_dirs = [
-      script_dir,
-      script_dir.parent,
-      Path.cwd(),
-   ]
-
-   for candidate_dir in candidate_dirs:
-      candidate_path = candidate_dir / config_file_name
-      if candidate_path.exists():
-         return candidate_path
-
-   return None
-
-
-def load_url(url: str | None, config_path: str | None) -> str:
-   if url:
-      return url
-
-   resolved_config_path = load_config_path(config_path)
-   if not resolved_config_path:
-      print(f"Config file not found; using default url: {DEFAULT_URL}")
-      return DEFAULT_URL
-
-   config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
-   dsl_api_server_config = config.get("DSLAPIServer", {})
-   ip = dsl_api_server_config.get("Ip", "localhost")
-   port = int(dsl_api_server_config.get("Port", 19093))
-   ws_url = f"ws://{ip}:{port}/ws/"
-   print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
-   return ws_url
-
-
-async def send_subscription(websocket, dsl: str) -> None:
-   payload = {
-      "type": "subscribe",
-      "dsl": dsl,
-   }
-   await websocket.send(json.dumps(payload))
-   print("Sent subscribe request:")
-   print(json.dumps(payload, indent=2))
-
-
-async def send_unsubscribe(websocket, subscription_id: str) -> None:
-   payload = {
-      "type": "unsubscribe",
-      "subscriptionId": subscription_id,
-   }
-   await websocket.send(json.dumps(payload))
-   print("Sent unsubscribe request:")
-   print(json.dumps(payload, indent=2))
-
-
-def print_server_message(raw_message: str) -> str | None:
-   try:
-      message = json.loads(raw_message)
-   except json.JSONDecodeError:
-      print(raw_message)
-      return None
-
-   print("Received server message:")
-   print(json.dumps(message, indent=2))
-   return message.get("subscriptionId")
-
-
-async def run_client(url: str, dsl: str) -> None:
-   subscription_id: str | None = None
-
-   async with websockets.connect(url) as websocket:
-      print(f"Connected to DSPublisher DSL websocket: {url}")
-      await send_subscription(websocket, dsl)
-
-      try:
-         async for raw_message in websocket:
-            latest_subscription_id = print_server_message(raw_message)
-            subscription_id = subscription_id or latest_subscription_id
-      finally:
-         if subscription_id:
-            await send_unsubscribe(websocket, subscription_id)
-
-
-def main() -> int:
-   parser = argparse.ArgumentParser(
-      description="Subscribe to a DSPublisher DSL websocket query and print responses."
-   )
-   parser.add_argument(
-      "--url",
-      help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
-   )
-   parser.add_argument(
-      "--config",
-      help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
-   )
-   parser.add_argument(
-      "--dsl",
-      help="DSL expression to subscribe to.",
-   )
-   parser.add_argument(
-      "--dsl-file",
-      help="Path to a text file containing the DSL expression.",
-   )
-   args = parser.parse_args()
-
-   try:
-      dsl = load_dsl(args.dsl, args.dsl_file)
-      url = load_url(args.url, args.config)
-      asyncio.run(run_client(url, dsl))
-      return 0
-   except KeyboardInterrupt:
-      print("stopped")
-      return 0
-   except Exception as ex:
-      print(f"error: {ex}")
-      return 1
-
-
-if __name__ == "__main__":
-   raise SystemExit(main())

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLAPIWSListener.cs

@@ -1,264 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Net;
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Threading;
-using System.Threading.Tasks;
-
-using BCG;
-using Bts.DataStreamCore;
-
-/// <summary>
-/// Websocket-facing API boundary for DSL clients. It opens the listener port,
-/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
-/// Instantiated by DSPAPIServer.
-/// </summary>
-public sealed class DSLAPIWSListener : IDisposable {
-   readonly HttpListener listener = new();
-   readonly IDSLBookkeeper bookkeeper;
-   readonly string prefix;
-   readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
-   CancellationTokenSource? cancellation_token_source;
-   Task? listener_task;
-   bool disposed;
-
-   sealed class ClientConnection {
-      public required string client_id { get; init; }
-      public required WebSocket web_socket { get; init; }
-      public HashSet<Guid> subscription_ids { get; } = new();
-      public SemaphoreSlim send_lock { get; } = new(1, 1);
-      public DataPacketJsonSerializer serializer { get; } = new();
-   }
-
-   public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
-      ArgumentNullException.ThrowIfNull(bookkeeper);
-      this.prefix = prefix;
-      this.bookkeeper = bookkeeper;
-   }
-
-   public Task StartAsync(CancellationToken cancellation_token = default) {
-      _throw_if_disposed();
-      if (listener_task is not null) {
-         return Task.CompletedTask;
-      }
-
-      cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
-      listener.Prefixes.Add(prefix);
-      listener.Start();
-      listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
-      return Task.CompletedTask;
-   }
-
-   public async Task StopAsync() {
-      if (cancellation_token_source is null) {
-         return;
-      }
-
-      cancellation_token_source.Cancel();
-      if (listener.IsListening) {
-         listener.Stop();
-      }
-
-      foreach (var client_connection in clients_by_id.Values) {
-         await _cleanup_client(client_connection);
-      }
-
-      if (listener_task is not null) {
-         await listener_task;
-         listener_task = null;
-      }
-
-      cancellation_token_source.Dispose();
-      cancellation_token_source = null;
-   }
-
-   public void Dispose() {
-      if (disposed) {
-         return;
-      }
-
-      StopAsync().GetAwaiter().GetResult();
-      listener.Close();
-      disposed = true;
-   }
-
-   async Task _listen_loop(CancellationToken cancellation_token) {
-      while (!cancellation_token.IsCancellationRequested) {
-         HttpListenerContext context;
-         try {
-            context = await listener.GetContextAsync();
-         } catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
-            break;
-         } catch (ObjectDisposedException) {
-            break;
-         }
-
-         _ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
-      }
-   }
-
-   async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
-      if (!context.Request.IsWebSocketRequest) {
-         context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
-         context.Response.Close();
-         return;
-      }
-
-      HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
-      var client_connection = new ClientConnection {
-         client_id = Guid.NewGuid().ToString("N"),
-         web_socket = web_socket_context.WebSocket,
-      };
-
-      clients_by_id[client_connection.client_id] = client_connection;
-      try {
-         await _read_messages(client_connection, cancellation_token);
-      } finally {
-         await _cleanup_client(client_connection);
-      }
-   }
-
-   async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
-      var buffer = new byte[64 * 1024];
-      while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
-         var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
-         if (result.MessageType == WebSocketMessageType.Close) {
-            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
-            break;
-         }
-
-         var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
-         await _handle_message(client_connection, json, cancellation_token);
-      }
-   }
-
-   async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
-      DSLWebSocketRequest? request;
-      try {
-         request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
-      } catch (JsonException ex) {
-         await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
-         return;
-      }
-
-      if (request?.type is null) {
-         await _send_error(client_connection, "Request type is required.", cancellation_token);
-         return;
-      }
-
-      if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
-         await _handle_subscribe(client_connection, request, cancellation_token);
-         return;
-      }
-
-      if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
-         await _handle_unsubscribe(client_connection, request, cancellation_token);
-         return;
-      }
-
-      await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
-   }
-
-   async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
-      if (string.IsNullOrWhiteSpace(request.dsl)) {
-         await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
-         return;
-      }
-
-      Guid subscription_id = Guid.Empty;
-      subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
-         if (subscription_id == Guid.Empty) {
-            return;
-         }
-         _ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
-      }, client_connection.client_id);
-
-      client_connection.subscription_ids.Add(subscription_id);
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "subscribed",
-         subscription_id = subscription_id,
-      }, cancellation_token);
-   }
-
-   async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
-      if (request.subscription_id is null) {
-         await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
-         return;
-      }
-
-      var subscription_id = request.subscription_id.Value;
-      if (!client_connection.subscription_ids.Remove(subscription_id)) {
-         await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
-         return;
-      }
-
-      bookkeeper.Unsubscribe(subscription_id);
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "unsubscribed",
-         subscription_id = subscription_id,
-      }, cancellation_token);
-   }
-
-   async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
-      var packet_json = client_connection.serializer.Serialize(packet);
-      using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "data",
-         subscription_id = subscription_id,
-         packet = packet_document.RootElement.Clone(),
-      }, cancellation_token);
-   }
-
-   Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
-      _send_response(client_connection, new DSLWebSocketResponse {
-         type = "error",
-         message = message,
-      }, cancellation_token);
-
-   async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
-      if (client_connection.web_socket.State != WebSocketState.Open) {
-         bookkeeper.DisposeClient(client_connection.client_id);
-         return;
-      }
-
-      var json = JsonSerializer.Serialize(response);
-      var bytes = Encoding.UTF8.GetBytes(json);
-      await client_connection.send_lock.WaitAsync(cancellation_token);
-      try {
-         await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
-      } catch {
-         bookkeeper.DisposeClient(client_connection.client_id);
-         throw;
-      } finally {
-         client_connection.send_lock.Release();
-      }
-   }
-
-   async Task _cleanup_client(ClientConnection client_connection) {
-      if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
-         bookkeeper.DisposeClient(client_connection.client_id);
-      }
-
-      if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
-         try {
-            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
-         } catch {
-         }
-      }
-
-      client_connection.send_lock.Dispose();
-      client_connection.web_socket.Dispose();
-   }
-
-   void _throw_if_disposed() {
-      if (disposed) {
-         throw new ObjectDisposedException(nameof(DSLAPIWSListener));
-      }
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLBookkeeper.cs

@@ -1,216 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Collections.Generic;
-
-using BCG;
-using Bts.DataStreamCore;
-using bts.BlueOrb;
-using bts;
-
-/// <summary>
-/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
-/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
-/// Instantiated by DSPAPIServer.
-/// </summary>
-public sealed class DSLBookkeeper : IDSLBookkeeper {
-   sealed class QueryState {
-      public required string dsl { get; init; }
-      public required DSLStream stream { get; init; }
-      public DataStream.ChangeDelegate? stream_handler { get; set; }
-      public HashSet<Guid> subscription_ids { get; } = new();
-      public Queue<DataPacket> pending_packets { get; } = new();
-   }
-
-   sealed class SubscriptionState {
-      public required Guid subscription_id { get; init; }
-      public required string client_id { get; init; }
-      public required string dsl_key { get; init; }
-      public required Action<DataPacket> on_data { get; init; }
-   }
-
-   readonly object lock_obj = new();
-   readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
-   readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
-   readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
-   bool disposed;
-
-   public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
-      ArgumentNullException.ThrowIfNull(on_data);
-      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
-
-      lock (lock_obj) {
-         _throw_if_disposed();
-
-         var dsl_key = _normalize_dsl(dsl);
-         if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
-            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
-            query_state = _create_query_state(dsl_key);
-            queries_by_dsl.Add(dsl_key, query_state);
-         } else {
-            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
-         }
-
-         var subscription_id = Guid.NewGuid();
-         var subscription_state = new SubscriptionState {
-            subscription_id = subscription_id,
-            client_id = client_id,
-            dsl_key = dsl_key,
-            on_data = on_data,
-         };
-
-         subscriptions_by_id.Add(subscription_id, subscription_state);
-         query_state.subscription_ids.Add(subscription_id);
-
-         if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
-            client_subscriptions = new();
-            subscriptions_by_client.Add(client_id, client_subscriptions);
-         }
-         client_subscriptions.Add(subscription_id);
-
-         _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
-         return subscription_id;
-      }
-   }
-
-   public void Unsubscribe(Guid subscription_id) {
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-         _unsubscribe_locked(subscription_id);
-      }
-   }
-
-   public void DisposeClient(string client_id) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
-
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
-            return;
-         }
-
-         foreach (var subscription_id in subscription_ids.ToArray()) {
-            _unsubscribe_locked(subscription_id);
-         }
-      }
-   }
-
-   public void DoWork() {
-      List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
-
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         foreach (var query_state in queries_by_dsl.Values) {
-            while (query_state.pending_packets.TryDequeue(out var packet)) {
-               foreach (var subscription_id in query_state.subscription_ids) {
-                  if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
-                     callbacks.Add((subscription_state.on_data, packet));
-                  }
-               }
-            }
-         }
-      }
-
-      foreach (var callback in callbacks) {
-         callback.on_data(callback.packet);
-      }
-   }
-
-   public void Dispose() {
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         foreach (var query_state in queries_by_dsl.Values) {
-            _dispose_query_state(query_state);
-         }
-
-         queries_by_dsl.Clear();
-         subscriptions_by_id.Clear();
-         subscriptions_by_client.Clear();
-         disposed = true;
-      }
-   }
-
-   QueryState _create_query_state(string dsl_key) {
-      _log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
-      var stream = new DSLStream(dsl_key);
-      var query_state = new QueryState {
-         dsl = dsl_key,
-         stream = stream,
-      };
-
-      DataStream.ChangeDelegate stream_handler = new_value => {
-         if (new_value is not DataPacket packet) {
-            return;
-         }
-
-         lock (lock_obj) {
-            if (!disposed) {
-               query_state.pending_packets.Enqueue(packet);
-            }
-         }
-      };
-
-      query_state.stream_handler = stream_handler;
-      stream.onChange += stream_handler;
-      return query_state;
-   }
-
-   void _unsubscribe_locked(Guid subscription_id) {
-      if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
-         return;
-      }
-
-      if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
-         client_subscriptions.Remove(subscription_id);
-         if (client_subscriptions.Count == 0) {
-            subscriptions_by_client.Remove(subscription_state.client_id);
-         }
-      }
-
-      if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
-         query_state.subscription_ids.Remove(subscription_id);
-         _log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
-         if (query_state.subscription_ids.Count == 0) {
-            _log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
-            _dispose_query_state(query_state);
-            queries_by_dsl.Remove(subscription_state.dsl_key);
-         }
-      }
-   }
-
-   static string _normalize_dsl(string dsl) =>
-      dsl.Trim();
-
-   static void _dispose_query_state(QueryState query_state) {
-      _log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
-      if (query_state.stream_handler is not null) {
-         query_state.stream.onChange -= query_state.stream_handler;
-         query_state.stream_handler = null;
-      }
-      query_state.subscription_ids.Clear();
-      query_state.pending_packets.Clear();
-   }
-
-   static void _log_lifecycle(string message) {
-      Console.WriteLine(message);
-      Log.Gen.info(message);
-   }
-
-   void _throw_if_disposed() {
-      if (disposed) {
-         throw new ObjectDisposedException(nameof(DSLBookkeeper));
-      }
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLWebSocketMessages.cs

@@ -1,39 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-/// <summary>
-/// Client-to-server websocket protocol message for subscribing and unsubscribing.
-/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
-/// </summary>
-public sealed class DSLWebSocketRequest {
-   [JsonPropertyName("type")]
-   public string? type { get; set; }
-
-   [JsonPropertyName("dsl")]
-   public string? dsl { get; set; }
-
-   [JsonPropertyName("subscriptionId")]
-   public Guid? subscription_id { get; set; }
-}
-
-/// <summary>
-/// Server-to-client websocket protocol message. Data responses carry serialized
-/// DataPacket JSON in the packet field.
-/// Instantiated by DSLAPIWSListener before serializing responses to clients.
-/// </summary>
-public sealed class DSLWebSocketResponse {
-   [JsonPropertyName("type")]
-   public string type { get; set; } = "";
-
-   [JsonPropertyName("subscriptionId")]
-   public Guid? subscription_id { get; set; }
-
-   [JsonPropertyName("message")]
-   public string? message { get; set; }
-
-   [JsonPropertyName("packet")]
-   public JsonElement? packet { get; set; }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSPAPIServer.cs

@@ -1,37 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-
-/// <summary>
-/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
-/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
-/// Instantiated by Program.cs.
-/// </summary>
-public sealed class DSPAPIServer : IDisposable {
-   readonly DSLBookkeeper bookkeeper;
-   readonly DSLAPIWSListener web_socket_listener;
-
-   public DSPAPIServer(global::DSLAPIServerConfig config) {
-      ArgumentNullException.ThrowIfNull(config);
-      bookkeeper = new DSLBookkeeper();
-      var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
-      web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
-      Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
-   }
-
-   public Task StartAsync(CancellationToken cancellation_token = default) =>
-      web_socket_listener.StartAsync(cancellation_token);
-
-   public Task StopAsync() =>
-      web_socket_listener.StopAsync();
-
-   public void DoWork() =>
-      bookkeeper.DoWork();
-
-   public void Dispose() {
-      web_socket_listener.Dispose();
-      bookkeeper.Dispose();
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/IDSLBookkeeper.cs

@@ -1,18 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-
-using BCG;
-using Bts.DataStreamCore;
-
-/// <summary>
-/// Internal contract between the websocket listener and the DSL subscription owner.
-/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
-/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
-/// </summary>
-public interface IDSLBookkeeper : IDisposable {
-   Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
-   void Unsubscribe(Guid subscription_id);
-   void DisposeClient(string client_id);
-   void DoWork();
-}

src/DataStreamSuite/DSPublisher/DSPublisher.csproj

@@ -1,31 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\PubSub\PubSub.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ConfigurationProfile Include="appsettings*.json" />
-
-    <None Update="appsettings*.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
-    </None>
-    <None Update="WSPublishers\**\*.py">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DSLWSClient\**\*.py">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/DSPublisher/DSPublisherConfig.cs

@@ -1,21 +0,0 @@
-using System.Collections.Generic;
-
-public sealed class DSPublisherConfig {
-   public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
-   public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
-}
-
-public sealed class DSLAPIServerConfig {
-   public string Ip { get; init; } = "localhost";
-   public int Port { get; init; } = 19093;
-}
-
-public sealed class StreamPublishingNode {
-   public string Name { get; init; } = string.Empty;
-   public string Expression { get; init; } = string.Empty;
-}
-
-public sealed class PythonPublishingNode {
-   public string PythonScriptLocation { get; init; } = string.Empty;
-   public string Args { get; init; } = string.Empty;
-}

src/DataStreamSuite/DSPublisher/Program.cs

@@ -1,125 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-using System.Threading;
-
-using BCG;
-using bts;
-using Bts.PubSub;
-using bts.BlueOrb;
-using DSPublisher.DSLWSServer;
-
-Console.WriteLine("Starting DSPublisher");
-
-var configPath = load_config_path();
-Console.WriteLine($"Loading config from: {configPath}");
-if (!File.Exists(configPath)) {
-   Console.WriteLine($"Config file not found: {configPath}");
-   return;
-}
-
-var json = File.ReadAllText(configPath);
-using var doc = JsonDocument.Parse(json);
-
-if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
-   Console.WriteLine("DSPublisher config section not found");
-   return;
-}
-
-var json_options = new JsonSerializerOptions {
-   PropertyNameCaseInsensitive = true,
-};
-
-var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
-var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
-
-Console.WriteLine("StreamPublishingNodes:");
-foreach (var node in config.StreamPublishingNodes) {
-   Console.WriteLine($"  Name: {node.Name}");
-   Console.WriteLine($"  Expression: {node.Expression}");
-}
-
-Console.WriteLine("PythonPublishingNodes:");
-foreach (var node in config.PythonPublishingNodes) {
-   Console.WriteLine($"  PythonScriptLocation: {node.PythonScriptLocation}");
-   Console.WriteLine($"  Args: {node.Args}");
-}
-
-var pubsub_config = load_pubsub_config(doc, json_options);
-PubSubConfig.SetGlobalConfig(pubsub_config);
-using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
-ViewStreamDatabase.SetGlobalInstance(view_stream_database);
-
-//var pub_table_broker = new PubTableBroker();
-//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
-
-using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
-await dsp_api_server.StartAsync();
-
-while (true) {
-   view_stream_database.doWork();
-   dsp_api_server.DoWork();
-   Thread.Sleep(TimeSpan.FromMilliseconds(10));
-}
-
-string load_config_path() {
-   var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
-   var baseFileName = string.IsNullOrWhiteSpace(environment)
-      ? "appsettings.json"
-      : $"appsettings.{environment}.json";
-   var dspPublisherFileName = $"DSPublisher.{baseFileName}";
-   var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
-   if (File.Exists(dspPublisherConfigPath)) {
-      return dspPublisherConfigPath;
-   }
-
-   return Path.Combine(AppContext.BaseDirectory, baseFileName);
-}
-
-PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
-   if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
-      Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
-      return PubSubConfig.Default();
-   }
-
-   var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
-   if (pubsub_config is null) {
-      Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
-      return PubSubConfig.Default();
-   }
-
-   Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
-   return pubsub_config;
-}
-
-DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
-   if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
-      Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
-      return new DSLAPIServerConfig();
-   }
-
-   var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
-   if (dsl_api_server_config is null) {
-      Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
-      return new DSLAPIServerConfig();
-   }
-
-   Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
-   return dsl_api_server_config;
-}
-
-List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
-   var stream_publishers = new List<IDisposable>();
-   foreach (var node in publishing_nodes) {
-      if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
-         Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
-         continue;
-      }
-
-      Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
-      var stream = new DSLStream(node.Expression);
-      stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
-   }
-
-   return stream_publishers;
-}

src/DataStreamSuite/DSPublisher/Services/.gitkeep

@@ -1 +0,0 @@
-

src/DataStreamSuite/DSPublisher/WSPublishers/DatabentoPublisher.py

@@ -1,146 +0,0 @@
-"""
-Requires:
-   pip install -U databento
-   pip install websockets
-
-Sources:
-   https://databento.com/docs/quickstart/set-up
-   https://databento.com/docs/schemas-and-data-formats/bbo
-"""
-
-import datetime as dt
-import subprocess
-import sys
-import time
-
-try:
-   import databento as db
-except ImportError:
-   print("Installing missing dependency: databento")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
-   import databento as db
-
-from databento.common.error import BentoError
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-from databento_transformer import DatabentoTransformer
-from pubsub_client import PubSubClient
-
-
-API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
-DATASET = "GLBX.MDP3"
-SYMBOL = "ES.FUT"
-PUBSUB_WS_URL = "ws://localhost:19092/ws/"
-BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
-TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
-DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
-
-_pubsub_client = PubSubClient(PUBSUB_WS_URL)
-_transformer = DatabentoTransformer(
-   symbol=SYMBOL,
-   bbo_table_name=BBO_TABLE_NAME,
-   trades_table_name=TRADES_TABLE_NAME,
-   definitions_table_name=DEFINITIONS_TABLE_NAME,
-)
-
-
-def _publish_contract_definitions() -> None:
-   historical_client = db.Historical(API_KEY)
-   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
-   schema_range = dataset_range.get("schema", {}).get("definition", {})
-   available_end = schema_range.get("end", dataset_range["end"])
-
-   if isinstance(available_end, str):
-      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
-   else:
-      available_end_dt = available_end
-
-   # Databento's range end is exclusive. Back off slightly so the request stays
-   # strictly inside the available window.
-   end_dt = available_end_dt - dt.timedelta(seconds=1)
-   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
-   start = start_dt.isoformat()
-   end = end_dt.isoformat()
-
-   definition_data = historical_client.timeseries.get_range(
-      dataset=DATASET,
-      schema="definition",
-      stype_in="parent",
-      symbols=SYMBOL,
-      start=start,
-      end=end,
-   )
-
-   df = definition_data.to_df()
-   payload = _transformer.build_definitions_payload(df)
-   if payload is None:
-      print("No Databento contract definitions returned")
-      return
-
-   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
-   _pubsub_client.publish(payload)
-
-
-def on_record(record) -> None:
-   # Print every callback first so we can see whether Databento is flowing
-   # even if our row extraction logic misses a field shape.
-   print(record)
-
-   payloads = _transformer.transform_record(record)
-   if payloads:
-      for payload in payloads:
-         _pubsub_client.publish(payload)
-      return
-
-   print("Skipping record with no BBO/trade fields")
-
-
-def main() -> None:
-   _pubsub_client.start()
-   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
-   try:
-      _publish_contract_definitions()
-   except Exception as ex:
-      print(f"Contract definition publish failed: {ex}", flush=True)
-
-   while True:
-      try:
-         client = db.Live(key=API_KEY)
-
-         client.subscribe(
-            dataset=DATASET,
-            schema="bbo-1s",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-         client.subscribe(
-            dataset=DATASET,
-            schema="trades",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-
-         client.add_callback(on_record)
-         client.start()
-         print("Databento client started, waiting for BBO and trade records...", flush=True)
-         client.block_for_close()
-         print("Databento client closed cleanly", flush=True)
-         break
-      except BentoError as ex:
-         print(f"Databento live connection error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-      except Exception as ex:
-         print(f"Databento publisher unexpected error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-
-
-if __name__ == "__main__":
-   main()

src/DataStreamSuite/DSPublisher/WSPublishers/databento_transformer.py

@@ -1,244 +0,0 @@
-import datetime as dt
-
-
-class DatabentoTransformer:
-   def __init__(
-      self,
-      symbol: str,
-      bbo_table_name: str,
-      trades_table_name: str,
-      definitions_table_name: str,
-   ):
-      self._symbol = symbol
-      self._bbo_table_name = bbo_table_name
-      self._trades_table_name = trades_table_name
-      self._definitions_table_name = definitions_table_name
-      self._instrument_id_to_symbol: dict[int, str] = {}
-      self._last_bbo_row: dict | None = None
-      self._trade_row_id = 0
-      self._trade_table_created = False
-
-   def transform_record(self, record) -> list[dict]:
-      payload = self._handle_bbo_record(record)
-      if payload is not None:
-         return [payload]
-
-      payload = self._handle_trade_record(record)
-      if payload is not None:
-         return [payload]
-
-      return []
-
-   def build_definitions_payload(self, definition_df) -> dict | None:
-      if definition_df is None or definition_df.empty:
-         return None
-
-      df = definition_df.reset_index()
-      if "instrument_class" in df.columns:
-         df = df[df["instrument_class"].astype(str) == "F"]
-      if "security_type" in df.columns:
-         df = df[df["security_type"].astype(str) == "FUT"]
-      if "raw_symbol" in df.columns:
-         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
-         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
-
-      rows: list[dict] = []
-      for idx, row in enumerate(df.itertuples(index=False), start=1):
-         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
-         security_name = str(getattr(row, "raw_symbol", ""))
-         if instrument_id > 0 and security_name:
-            self._instrument_id_to_symbol[instrument_id] = security_name
-
-         rows.append({
-            "id": idx,
-            "security_name": security_name,
-            "instrument_id": instrument_id,
-            "instrument_class": str(getattr(row, "instrument_class", "")),
-            "security_type": str(getattr(row, "security_type", "")),
-            "asset": str(getattr(row, "asset", "")),
-            "exchange": str(getattr(row, "exchange", "")),
-            "currency": str(getattr(row, "currency", "")),
-            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
-            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
-            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
-            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
-            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
-            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
-            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
-            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
-            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
-            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
-            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
-            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
-            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
-            "underlying": str(getattr(row, "underlying", "")),
-         })
-
-      if not rows:
-         return None
-
-      return {
-         "op": "create_table",
-         "table_name": self._definitions_table_name,
-         "rows": rows,
-      }
-
-   def _handle_bbo_record(self, record) -> dict | None:
-      row = self._to_bbo_row(record)
-      if row is None:
-         return None
-
-      if self._last_bbo_row is None:
-         payload = {
-            "op": "create_table",
-            "table_name": self._bbo_table_name,
-            "rows": [row],
-         }
-      else:
-         payload = {
-            "op": "multiple_remove_add",
-            "table_name": self._bbo_table_name,
-            "removed_rows": [self._last_bbo_row],
-            "added_rows": [row],
-         }
-
-      self._last_bbo_row = row
-      return payload
-
-   def _handle_trade_record(self, record) -> dict | None:
-      row = self._to_trade_row(record)
-      if row is None:
-         return None
-
-      row["id"] = self._trade_row_id
-      self._trade_row_id += 1
-
-      if not self._trade_table_created:
-         payload = {
-            "op": "create_table",
-            "table_name": self._trades_table_name,
-            "rows": [row],
-         }
-         self._trade_table_created = True
-         return payload
-
-      return {
-         "op": "multiple_remove_add",
-         "table_name": self._trades_table_name,
-         "removed_rows": [],
-         "added_rows": [row],
-      }
-
-   def _to_bbo_row(self, record) -> dict | None:
-      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
-      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
-      if bid_px is None and ask_px is None:
-         return None
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "bid_px": float(bid_px or 0.0),
-         "ask_px": float(ask_px or 0.0),
-         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
-         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
-      }
-
-   def _to_trade_row(self, record) -> dict | None:
-      price = self._get_attr(record, "price", "px")
-      size = self._get_attr(record, "size")
-      if price is None and size is None:
-         return None
-
-      side = self._get_attr(record, "side", "aggressor_side", default="")
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "price": float(price or 0.0),
-         "size": int(size or 0),
-         "side": str(side),
-      }
-
-   def _get_security_name(self, record) -> str:
-      instrument_id = self._get_attr(record, "instrument_id", default=0)
-      try:
-         instrument_id_int = int(instrument_id or 0)
-      except (TypeError, ValueError):
-         instrument_id_int = 0
-
-      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
-         return self._instrument_id_to_symbol[instrument_id_int]
-
-      value = self._get_attr(
-         record,
-         "symbol",
-         "raw_symbol",
-         "stype_out_symbol",
-         "stype_in_symbol",
-         default=self._symbol,
-      )
-      return str(value)
-
-   def _get_timestamp(self, record) -> int:
-      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
-      try:
-         return int(value)
-      except (TypeError, ValueError):
-         return 0
-
-   def _get_attr(self, record, *names, default=None):
-      for name in names:
-         if hasattr(record, name):
-            value = getattr(record, name)
-            if value is not None:
-               return value
-      return default
-
-   def _get_level_attr(self, record, name, default=None):
-      levels = self._get_attr(record, "levels", default=None)
-      if levels:
-         level0 = levels[0]
-         if hasattr(level0, name):
-            value = getattr(level0, name)
-            if value is not None:
-               return value
-      return default
-
-   def _ns_to_iso(self, timestamp_ns) -> str:
-      if timestamp_ns is None:
-         return ""
-
-      if hasattr(timestamp_ns, "isoformat"):
-         try:
-            if hasattr(timestamp_ns, "to_pydatetime"):
-               timestamp_dt = timestamp_ns.to_pydatetime()
-            else:
-               timestamp_dt = timestamp_ns
-
-            if timestamp_dt is None:
-               return ""
-
-            if getattr(timestamp_dt, "tzinfo", None) is None:
-               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
-            return timestamp_dt.isoformat()
-         except (TypeError, ValueError, AttributeError):
-            pass
-
-      try:
-         ts_int = int(timestamp_ns)
-         if ts_int <= 0:
-            return ""
-         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
-      except (TypeError, ValueError, OSError, OverflowError):
-         return ""
-
-   def _nano_to_float(self, value) -> float:
-      try:
-         return float(value) / 1_000_000_000.0
-      except (TypeError, ValueError):
-         return 0.0

src/DataStreamSuite/DSPublisher/WSPublishers/pubsub_client.py

@@ -1,58 +0,0 @@
-import asyncio
-import json
-import queue
-import threading
-
-import websockets
-
-
-class PubSubClient:
-   def __init__(self, ws_url: str):
-      self._ws_url = ws_url
-      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
-      self._sender_thread: threading.Thread | None = None
-
-   def start(self) -> threading.Thread:
-      if self._sender_thread is None or not self._sender_thread.is_alive():
-         self._sender_thread = self._start_sender_thread()
-      return self._sender_thread
-
-   def publish(self, payload: dict) -> None:
-      self._outgoing_queue.put(payload)
-
-   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
-      self.publish({
-         "op": "create_table",
-         "table_name": table_name,
-         "rows": rows,
-      })
-
-   def publish_multiple_remove_add(
-      self,
-      table_name: str,
-      removed_rows: list[dict],
-      added_rows: list[dict],
-   ) -> None:
-      self.publish({
-         "op": "multiple_remove_add",
-         "table_name": table_name,
-         "removed_rows": removed_rows,
-         "added_rows": added_rows,
-      })
-
-   def _start_sender_thread(self) -> threading.Thread:
-      thread = threading.Thread(target=self._run_send_loop, daemon=True)
-      thread.start()
-      return thread
-
-   def _run_send_loop(self) -> None:
-      asyncio.run(self._ws_send_loop())
-
-   async def _ws_send_loop(self) -> None:
-      async with websockets.connect(self._ws_url) as ws:
-         print(f"Connected to PubSubServer websocket: {self._ws_url}")
-         while True:
-            payload = await asyncio.to_thread(self._outgoing_queue.get)
-            body = json.dumps(payload)
-            print(body)
-            await ws.send(body)

src/DataStreamSuite/DSPublisher/appsettings.Development.json

@@ -1,69 +0,0 @@
-{
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
-  "DSLAPIServer": {
-    "Ip": "localhost",
-    "Port": 19093
-  },
-  "DSPublisher": {
-    "StreamPublishingNodes": [
-      {
-        "Name": "sample_stream",
-        "Expression": "MillisecondStream(1000,5)"
-      }
-    ],
-    "PythonPublishingNodes": [
-      {
-        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
-        "Args": ""
-      }
-    ]
-  }
-}

src/DataStreamSuite/DSPublisher/appsettings.json

@@ -1,69 +0,0 @@
-{
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
-  "DSLAPIServer": {
-    "Ip": "localhost",
-    "Port": 19093
-  },
-  "DSPublisher": {
-    "StreamPublishingNodes": [
-      {
-        "Name": "sample_stream",
-        "Expression": "MillisecondStream(1000,5)"
-      }
-    ],
-    "PythonPublishingNodes": [
-      {
-        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
-        "Args": ""
-      }
-    ]
-  }
-}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -2,8 +2,7 @@
    using BCG;
 
    [DataStreamUserRespresentation("AddColumnsStream")]
-   // example: AddColumnsStream(l,"jc1:001")
-   public class AddColumnsStream : ViewStream {
+   public class AddColumnsStream: ViewStream {
       public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
          ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/HideColumnsStream.cs

@@ -1,126 +0,0 @@
-namespace BlueOrb.Operators.Core {
-   using BCG;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-
-   [DataStreamUserRespresentation("HideColumnsStream")]
-   public class HideColumnsStream : ViewStream {
-      // Wrapper row to hide the columns
-      public class MyRow : IBlueRow {
-         #region IBlueRow members
-         private IBlueTableSchema _schema;
-         private IBlueRow _row;
-         public MyRow(IBlueTableSchema schema, IBlueRow row) {
-            _schema = schema;
-            _row = row;
-         }
-         public IBlueTableSchema schema { get { return _schema; } }
-         #endregion
-
-         #region ICollection members
-         public int Count { get { return schema.columnNames.Length; } }
-
-         public bool IsSynchronized { get { return false; } }
-
-         public object SyncRoot { get { return this; } }
-
-         public void CopyTo(Array array, int index) {
-            int i = index;
-            foreach (object o in this as IEnumerable)
-               array.SetValue(o, i++);
-         }
-         #endregion
-
-         #region IDictionary members
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public object this[object key] {
-            get {
-               if (key is string)
-                  return this[(string)key];
-               return this[(int)key];
-            }
-            set { throw new NotSupportedException("The IBlueRow is readonly"); }
-         }
-
-         public object this[int index] {
-            get {
-               return this[schema.columnInfo[index].colName];
-            }
-         }
-
-         public object this[string col_name] {
-            get {
-               if (_schema.contains_column(col_name))
-                  return _row[col_name];
-               else
-                  throw new ApplicationException($"{col_name} not found");
-            }
-         }
-
-         public ICollection Keys { get { return schema.columnNames; } }
-
-         public ICollection Values {
-            get {
-               object[] retVal = new object[schema.columnNames.Length];
-               CopyTo(retVal, 0);
-               return retVal;
-            }
-         }
-
-         public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-         public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-         public bool Contains(object key) { return schema[(string)key] != null; }
-
-         public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-         #endregion
-
-      }
-
-      private readonly ViewStream _parent_str;
-      private readonly HashSet<string> _removed_cols;
-      private BlueTable _state;
-      public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
-         _parent_str = parent_str;
-         _removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
-         if (_removed_cols.Contains("id"))
-            _removed_cols.Remove("id");
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _handle_parent_str_cb;
-      }
-
-      private void _handle_parent_str_cb(object dp_obj) {
-         // this could be made incremental and hence faster
-         DataPacket dp_in = dp_obj as DataPacket;
-         if (_state == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
-               if (!_removed_cols.Contains(ci.colName))
-                  cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
-            }
-            _state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
-         }
-         _state.clear();
-         foreach (IBlueRow r in dp_in.newRows) {
-            _state.addRow(new MyRow(_state.schema, r));
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_state,
-            myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
-         myFireEvent(dp_out);
-      }
-   }
-
-
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/CartesianProductStream.cs

@@ -1,14 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("CartesianProductStream")]
-   public class CartesianProductStream : ViewStream {
-      public CartesianProductStream(ViewStream ls, ViewStream rs) {
-         ls = new AddColumnsStream(ls, "ljc: 001");
-         rs = new AddColumnsStream(rs, "rjc: 001");
-         ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
-         res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
-         mySetRealSource(res);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/FullJoinStream.cs

@@ -1,15 +0,0 @@
-namespace BlueOrb.Operators.Core {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   // JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,5 +1,4 @@
-namespace bts.BlueOrb
-{
+namespace bts.BlueOrb {
    using BCG;
 
    using Bts.DataStream;
@@ -14,10 +13,8 @@
    using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream
-   {
-      public class JoinBookkeepingData
-      {
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
          // initialized in cons
          readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
@@ -26,8 +23,7 @@
          int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
 
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
-         {
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
             _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
@@ -44,28 +40,22 @@
          public List<IBlueRow>? get_matching_rows(int hash) =>
             _hash_row_map.GetValueOrDefault(hash);
 
-         private RowHash _get_hash(IBlueRow r)
-         {
-            if (_join_cols == null || _join_cols.Length == 0)
-            {
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(IDataPacket dp)
-         {
-            if (!ready)
-            {
+         public void update_data(IDataPacket dp) {
+            if (!ready) {
                _schema = dp.newRows.schema;
                _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows)
-            {
+            foreach (IBlueRow r in dp.newRows) {
                int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
-               {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -74,10 +64,8 @@
          }
 
          [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready
-         {
-            get
-            {
+         public bool ready {
+            get {
                return _schema != null;
             }
          }
@@ -89,11 +77,9 @@
       IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
-      { }
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
 
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
-      {
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
          _ls = ls;
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
@@ -105,35 +91,30 @@
          _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _ls.onChange += _ls_onchange;
          _rs.onChange += _rs_onchange;
       }
 
-      private void _ls_onchange(object dp_obj)
-      {
+      private void _ls_onchange(object dp_obj) {
          var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
-      private void _rs_onchange(object dp_obj)
-      {
+      private void _rs_onchange(object dp_obj) {
          var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _ls.onChange -= _ls_onchange;
          _rs.onChange -= _rs_onchange;
       }
 
       [MemberNotNull(nameof(_schema))]
-      private void _build_schema()
-      {
+      private void _build_schema() {
          if (_schema != null)
             return;
          var lschema = Log.notnull(_ldata.schema);
@@ -141,64 +122,52 @@
          var cis = new List<ColumnInfo> {
             new("id", 0, TypeCode.Int32)
          };
-         foreach (ColumnInfo ci in lschema.columnInfo)
-         {
+         foreach (ColumnInfo ci in lschema.columnInfo) {
             string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in rschema.columnInfo)
-         {
+         foreach (ColumnInfo ci in rschema.columnInfo) {
             string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
          _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
-      private void _form_and_fire()
-      {
+      private void _form_and_fire() {
          if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
 
          // this is where the join happens
-         foreach (var kv in _ldata.rows)
-         {
+         foreach (var kv in _ldata.rows) {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
             List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows)
-            {
-               if (r_rows != null)
-               {
-                  foreach (IBlueRow r in r_rows)
-                  {
+            foreach (IBlueRow lr in l_rows) {
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
                      copy_row(rowvals, r, false);
                      res_rows.Add(rowvals);
                   }
-               }
-               else
-               {
+               } else {
                   object[] rowvals = build_row_array(res_rows.Count, lr);
                   res_rows.Add(rowvals);
                }
             }
          }
          BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows)
-         {
+         foreach (object[] rval in res_rows) {
             rv.addRow(new BlueRow(_schema, rval));
          }
          var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
 
-         object[] build_row_array(int id, IBlueRow l_row)
-         {
+         object[] build_row_array(int id, IBlueRow l_row) {
             object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++)
-            {
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
                rv[i] = DBNull.Value;
             }
             rv[0] = id;
@@ -206,26 +175,30 @@
             return rv;
          }
 
-         void copy_row(object?[] rval, IBlueRow r, bool is_left)
-         {
+         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
             int offset = 1;
             JoinBookkeepingData sdata = _ldata;
-            if (is_left)
-            {
+            if (is_left) {
                offset = 1;
                sdata = _ldata;
-            }
-            else
-            {
+            } else {
                sdata = _rdata;
                offset = 1 + _ldata.RequireSchema().columnInfo.Length;
             }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
-            {
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
                rval[offset + i] = r[i];
             }
          }
       }
    }
 
+   [DataStreamUserRespresentation("FullJoinStream")]
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+
 }

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -15,7 +15,7 @@ namespace bts.BlueOrb {
       private int _size = 0;
 
       public TimeSeriesStream(ViewStream stream)
-         : this(stream, Int32.MaxValue) { }
+         : this(stream, 1000) { }
 
       public TimeSeriesStream(ViewStream stream, int size) {
          _stream = stream;
@@ -29,22 +29,18 @@ namespace bts.BlueOrb {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
-      private bool _orig_schema_has_timestamp = false;
+
       private void _onstream_change(object o) {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
          if (_table == null) {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            if (!schema.contains_column("timestamp")) {
-               _orig_schema_has_timestamp = true;
-               schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            }
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
             _table = new CircularBufferTable(schema, _size);
          }
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            if (!_orig_schema_has_timestamp)
-               ar.setValue("timestamp", now);
+            ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
             _table.add_row(ar);
          }

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -23,16 +23,16 @@ public static class DataStreamExtensions {
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
       source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
 
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
+      IDisposable Subscribe(IObserver<IDataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
          void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
+               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
 

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -88,8 +88,7 @@ namespace BCG {
          return Task.CompletedTask;
       }
 
-      private void _on_str(DataPacket dp)
-      {
+      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");

src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs

@@ -0,0 +1,22 @@
+// using System;
+// using BCG;
+// using BCG.Utilities;
+
+// namespace BCG
+// {
+//    public class CartesianProductStream : ViewStream
+//    {
+//       public CartesianProductStream(ViewStream v1, ViewStream v2)
+//       {
+//          mySetRealSource(
+//          new JoinStream (new AddColumnsStream (v1,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn dateTime:dateBoundary",
+//             new AddColumnsStream (v2,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn"));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -14,10 +14,8 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
          : base (colMapExpression, null, columnNames) {}
 
-
-      public override IBlueRow evaluate (DataPacket packet)
-      {
-         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
+      public override IBlueRow? evaluate(IDataPacket packet) {
+         myOutputTable ??= new GroupByOutputTable(packet.Schema);
          string hashString = GroupByStream.getRepString (packet, myGroupByCols);
          if (packet.newRows.Count == 0)
             //if there are no rows for this group we remove the group from the output table

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -42,7 +42,7 @@ namespace BCG
          return retVal;
       }
 
-      public abstract IBlueRow evaluate (DataPacket dp) ;
+      public abstract IBlueRow? evaluate(IDataPacket dp);
       #endregion
 
       #region Helper methods

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -8,12 +8,13 @@ namespace BCG {
    internal class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
-         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         readonly Func<IDataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
             _groupby_func = groupby_func;
          }
-         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
+         public IBlueRow evaluate(IDataPacket packet) =>
+            _groupby_func(packet);
 
          public string[] groupByColumns { get { return _groupby_cols; } }
 
@@ -32,7 +33,7 @@ namespace BCG {
       private IBlueTableSchema myOutputSchema;
       private int myIdGenerator = 0;
 
-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
          : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
       public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -187,11 +188,11 @@ namespace BCG {
          }
       }
 
-      public static string getRepString(DataPacket dp, string[] colNames) {
+      public static string getRepString(IDataPacket dp, string[] colNames) {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow(IDataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)

src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
    /// <summary>
@@ -14,7 +12,7 @@ namespace BCG
       /// deletetion) and the added and deletedRows being the increments. If a group has
       /// to be totally removed then a null should be returned
       /// </summary>
-      IBlueRow evaluate (DataPacket packet);
+      IBlueRow? evaluate(IDataPacket packet);
 
       /// <summary>
       /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -14,9 +14,8 @@ namespace BCG {
         public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMaxRow = myOutputTable.getRow(hashString);
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,7 +1,3 @@
-using System;
-
-using BCG.Utilities;
-
 namespace BCG {
     public class MinGroupByEvaluator : MaxGroupByEvaluator {
         public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -10,8 +6,8 @@ namespace BCG {
         public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
            base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMinRow = myOutputTable.getRow(stringRep);
             IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG {
     /// <summary>
     /// Computes the sum for a given column
@@ -10,8 +8,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
 
             IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
             return sumRow;
@@ -33,7 +31,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 
@@ -51,9 +49,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
             return sumRow;
         }
@@ -68,7 +65,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -2,6 +2,8 @@ using BCG;
 
 using MongoDB.Driver.Core.WireProtocol.Messages;
 
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -37,7 +39,7 @@ namespace bts {
       }
 
       void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
          IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
          BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
          :

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -8,34 +8,48 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using bts;
-using BCG;
 
-namespace BlueOrb.Pubsub.Diagnostics {
-   // This publishes data to the pubsub server, subscribes, gets it back and records the time it took for the round trip
+namespace bts
+{
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
          public readonly string? msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[]? payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
                rv[i] = payload_byte;
             }
             return rv;
          }
 
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
+         // {
+         //    guid = guid_;
+         //    timestamp = timestamp_;
+         //    payload = new byte[payload_size];
+         //    for (int i = 0; i < payload_size; i++)
+         //    {
+         //       payload[i] = payload_byte;
+         //    }
+         // }
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id ?? string.Empty);
@@ -47,7 +61,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
             var str = new MemoryStream(ser_bytes) {
                Position = 0
             };
@@ -72,7 +87,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
          SEND = 0,
          RECV = 1
       };
@@ -107,7 +123,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _pub_core = PublisherCore.Create(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -115,7 +132,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
 
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _vsd.removeWorker(this);
          Log.notnull(_sub_core).unsubscribe(_tag_name);
       }
@@ -126,7 +144,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
             SubscriberRoutes = new() { new(serverUrl, ".*") },
          });
 
-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
          Log.notnull(_last_sent_msg);
          // as soon as the data is published, its reflected in the stream state
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -136,7 +155,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
          Log.notnull(_last_recvd_msg);
          Log.notnull(_last_sent_msg);
          // update the last publish row
@@ -153,11 +173,13 @@ namespace BlueOrb.Pubsub.Diagnostics {
          }
       }
 
-      private string _get_next_id() {
+      private string _get_next_id()
+      {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
          Log.notnull(_pub_core);
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
@@ -166,10 +188,13 @@ namespace BlueOrb.Pubsub.Diagnostics {
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork() {
-         switch (_state) {
+      public void doWork()
+      {
+         switch (_state)
+         {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -181,7 +206,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
                var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -204,7 +230,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
          }
       }
 
-      public static long epoch_now() {
+      public static long epoch_now()
+      {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -1,17 +1,12 @@
 namespace bts {
    using BCG;
 
-   using bts.collections;
-
-   using IronPython.Runtime.Operations;
-
    using ProtoBuf;
 
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
-   using System.Runtime.Intrinsics.Arm;
 
    public class ProjectData {
       // schema from the base stream

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -1,5 +1,4 @@
-namespace BlueOrb.PubSub {
-   using BCG;
+namespace BCG {
    // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
    // and hence epublishes it out to the PS
    public class PublisherStream : ViewStream {

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -15,7 +15,6 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
 using bts;
@@ -360,11 +359,11 @@ namespace BCG {
       }
 
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
-         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         Dictionary<string, string> dictionary = JsonSerializer.Deserialize<Dictionary<string, string>>(json_string);
          return dictionary;
       }
       public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
-         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         Dictionary<string, object> dictionary = JsonSerializer.Deserialize<Dictionary<string, object>>(json_string);
          if (!dictionary.ContainsKey("id")) {
             dictionary.Add("id", id);
          }
@@ -1847,7 +1846,7 @@ namespace BCG {
          return sb.ToString();
       }
 
-      public static string rows_id_string(this IBlueRowCollection table) {
+      public static string id_string(IBlueRowCollection table) {
          string rv = String.Empty;
          foreach (IBlueRow r in table)
             rv += $"{r["id"]} ";
@@ -2246,7 +2245,7 @@ namespace BCG {
 
       public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
          try {
-            object rv = (r != null && r.schema.contains_column(col_name)) ? r[col_name] : DBNull.Value;
+            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
             if (rv == null || rv == DBNull.Value)
                return error_val;
             else

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,4 +1,6 @@
 #nullable enable
+using Bts.DataStreamCore;
+
 using System;
 using System.Text.Json.Serialization;
 
@@ -7,6 +9,7 @@ namespace BCG {
    /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
    /// </summary>
    [Serializable]
+   [JsonConverter(typeof(ColumnInfoJsonConverter))]
    public class ColumnInfo : IComparable<ColumnInfo>, IComparable, IEquatable<ColumnInfo> {
 
       //the default column info does not cach values in a lookup table, therefore it is in group 0
@@ -67,16 +70,12 @@ namespace BCG {
       private Type myColType;
       private object? myColValue;
 
-      [JsonPropertyName("colNo")]
       public int colNo { get { return myColNo; } }
 
-      [JsonPropertyName("colName")]
       public string colName { get { return myColName; } }
 
-      [JsonPropertyName("colType")]
       public Type colType { get { return myColType; } }
 
-      [JsonIgnore]
       public object? colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfoJsonConverter.cs

@@ -0,0 +1,27 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class ColumnInfoJsonConverter : JsonConverter<ColumnInfo> {
+   public override ColumnInfo Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      var jobj = JsonSerializer.Deserialize<JsonObject>(ref reader);
+      return new(
+         jobj[nameof(ColumnInfo.colName)].GetValue<string>(),
+         jobj[nameof(ColumnInfo.colNo)].GetValue<int>(),
+         Type.GetType(jobj[nameof(ColumnInfo.colType)].GetValue<string>()));
+   }
+
+   public override void Write(Utf8JsonWriter writer, ColumnInfo value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      writer.WriteString(nameof(ColumnInfo.colName), value.colName);
+      writer.WriteNumber(nameof(ColumnInfo.colNo), value.colNo);
+      writer.WriteString(nameof(ColumnInfo.colType), value.colType.ToString());
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/Comparisons.cs

@@ -1,142 +0,0 @@
-namespace BlueOrb.Data {
-   using BCG;
-
-   using bts;
-
-
-   using System;
-   using System.Linq;
-
-   public static class Comparisons {
-      public static int compare_row(this IBlueRow row, IBlueRow other_row, params string[] col_names) {
-         int comp_res = 0;
-         foreach (string col_name in col_names) {
-            object val1 = row[col_name];
-            object val2 = other_row[col_name];
-            comp_res = compare_row_values(val1, val2);
-            if (comp_res != 0)
-               break;
-         }
-         return comp_res;
-      }
-
-      public static void compare_tables_id(this IBlueRowCollection table1,
-                                           IBlueRowCollection table2,
-                                           out IBlueRowCollection rows_only_in_table1,
-                                           out IBlueRowCollection rows_only_in_table2) {
-         Log.assert(table1 != null);
-         table2 = table2 ?? new BlueTable(table1.schema);
-         var schema = get_output_schema(table1, table2);
-         var table1_only = new BlueTable(schema);
-         var table2_only = new BlueTable(schema);
-
-         if (table1 is not null) {
-            foreach (IBlueRow row in table1)
-               if (table2 is null || !table2.containsID(row.get<int>("id")))
-                  table1_only.addRow(row);
-         }
-
-         if (table2 is not null) {
-            foreach (IBlueRow row in table2)
-               if (table1 is null || !table1.containsID(row.get<int>("id")))
-                  table2_only.addRow(row);
-         }
-
-         rows_only_in_table1 = table1_only;
-         rows_only_in_table2 = table2_only;
-      }
-
-      public static void compare_tables_deep(this IBlueRowCollection table1,
-                                             IBlueRowCollection table2,
-                                             out IBlueRowCollection rows_only_in_table1,
-                                             out IBlueRowCollection rows_only_in_table2) {
-         Log.assert(table1 != null);
-         table2 = table2 ?? new BlueTable(table1.schema);
-         var schema = get_output_schema(table1, table2);
-         validate_matching_schemas(table1, table2);
-
-         var table1_only = new BlueTable(schema);
-         var table2_only = new BlueTable(schema);
-         var compare_cols = schema.columnNames.Where(c => c != "id").ToArray();
-
-         if (table1 is not null) {
-            foreach (IBlueRow row in table1) {
-               if (table2 is null || !table2.containsID(row.get<int>("id"))) {
-                  table1_only.addRow(row);
-                  continue;
-               }
-
-               var other_row = table2[row.get<int>("id")];
-               if (row.compare_row(other_row, compare_cols) != 0) {
-                  table1_only.addRow(row);
-                  table2_only.addRow(other_row);
-               }
-            }
-         }
-
-         if (table2 is not null) {
-            foreach (IBlueRow row in table2)
-               if (table1 is null || !table1.containsID(row.get<int>("id")))
-                  table2_only.addRow(row);
-         }
-
-         rows_only_in_table1 = table1_only;
-         rows_only_in_table2 = table2_only;
-      }
-
-      static int compare_row_values(object v1, object v2) {
-         if (is_null(v1))
-            return is_null(v2) ? 0 : -1;
-         if (is_null(v2))
-            return 1;
-         try {
-            var comparable1 = (IComparable)v1;
-            return comparable1.CompareTo(v2);
-         } catch (InvalidCastException) {
-            if (v1 is byte[] bytes1 && v2 is byte[] bytes2)
-               return compare_bytes(bytes1, bytes2) ? 0 : 1;
-            throw;
-         }
-      }
-
-      static bool compare_bytes(byte[] v1, byte[] v2) {
-         if (v1.Length != v2.Length)
-            return false;
-
-         for (int i = 0; i < v1.Length; i++)
-            if (v1[i] != v2[i])
-               return false;
-
-         return true;
-      }
-
-      static bool is_null(object o) {
-         return o == null || o == DBNull.Value;
-      }
-
-      static IBlueTableSchema get_output_schema(IBlueRowCollection table1, IBlueRowCollection table2) {
-         if (table1 is not null)
-            return table1.schema;
-         if (table2 is not null)
-            return table2.schema;
-         throw new ArgumentNullException(nameof(table1), "At least one table must be non-null.");
-      }
-
-      static void validate_matching_schemas(IBlueRowCollection table1, IBlueRowCollection table2) {
-         if (table1 is null || table2 is null)
-            return;
-
-         var schema1 = table1.schema;
-         var schema2 = table2.schema;
-         if (schema1.columnInfo.Length != schema2.columnInfo.Length)
-            throw new ArgumentException("Tables must have matching schemas.");
-
-         for (int i = 0; i < schema1.columnInfo.Length; i++) {
-            var c1 = schema1.columnInfo[i];
-            var c2 = schema2.columnInfo[i];
-            if (c1.colName != c2.colName || c1.colType != c2.colType)
-               throw new ArgumentException("Tables must have matching schemas.");
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/ToStrings.cs

@@ -1,35 +0,0 @@
-namespace BlueOrb.Data {
-   using BCG;
-
-   using System;
-   using System.Text;
-
-public static class ToStrings {
-      public static string ToDebugString(this IBlueRow row) {
-         StringBuilder sb = new StringBuilder();
-         foreach (ColumnInfo c in row.schema.columnInfo) {
-            if (sb.Length != 0)
-               sb.Append(", ");
-            sb.Append(c.colName);
-            sb.Append(": ");
-            object val = row[c.colName];
-            sb.Append(is_null(val) ? "NULL" : val.ToString());
-         }
-         return sb.ToString();
-      }
-
-      public static string ToDebugString(this IBlueRowCollection rows) {
-         StringBuilder sb = new StringBuilder();
-         foreach (IBlueRow row in rows) {
-            if (sb.Length != 0)
-               sb.AppendLine();
-            sb.Append(row.ToDebugString());
-         }
-         return sb.ToString();
-      }
-
-      static bool is_null(object o) {
-         return o == null || o == DBNull.Value;
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -41,7 +41,7 @@ namespace BCG {
       }
 
       public IBlueRowCollection newRows, addedRows, deletedRows;
-      public IDictionary<string, object> meta_data = null;
+      public Dictionary<string, object> meta_data = null;
       public DataPacket() { }
 
       public DataPacket(IBlueTableSchema schema)
@@ -53,7 +53,7 @@ namespace BCG {
          this.deletedRows = deletedRows;
       }
       public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows,
-                  IDictionary<string, object> meta_data) {
+                  Dictionary<string, object> meta_data) {
          this.newRows = newRows;
          this.addedRows = addedRows;
          this.deletedRows = deletedRows;
@@ -63,6 +63,7 @@ namespace BCG {
       IBlueRowCollection IDataPacket.newRows => newRows;
       IBlueRowCollection IDataPacket.addedRows => addedRows;
       IBlueRowCollection IDataPacket.deletedRows => deletedRows;
+      IReadOnlyDictionary<string, object> IDataPacket.Metadata => meta_data;
 
       public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
          new_rows = newRows;

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -8,7 +8,6 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="OpenAI" Version="2.0.0" />
     <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
   </ItemGroup>

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -6,89 +6,89 @@ namespace BCG {
    using System.Reflection;
    using System.IO;
    using Bts.DataStreamCore;
-   using System.Collections.Generic;
-   using System.Linq;
 
    public class PrimitiveManager {
       static MarketDataStreamFactory mdStreamFactory;
-      private Dictionary<string, Type> _dslopname_to_type_map = new Dictionary<string, Type>();
-      private static PrimitiveManager _self;
-      private HashSet<string> _loaded_assemblies = new HashSet<string>();
-      private static string[] _allowed_namespaces_matches = new string[] {
-         "blueorb"
-      };
+      private IDictionary myTypeLookupTable = new Hashtable();
+      private static PrimitiveManager mySelf;
+      private ArrayList myAssemblyNames = new ArrayList();
 
       private PrimitiveManager() {
-         _load_all_streams();
+         myLoadAllTypes();
       }
 
-      private static bool _is_loadable_stream(Type type) {
-         Func<bool> namespace_match = () => {
-            foreach (string ns in _allowed_namespaces_matches) {
-               if (type.Namespace.ToLower().Contains(ns.ToLower()))
-                  return true;
-            }
-            return false;
-         };
-         bool rv = type.IsSubclassOf(typeof(ViewStream))
-                   && type.IsPublic;
-         //rv = rv && (!type.IsInterface && !type.IsAbstract);
-         rv = rv && namespace_match();
-         return rv;
-      }
-
-      private void _load_all_streams() {
-         load_streams_from_assembly(this.GetType().Assembly.FullName);
+      private void myLoadAllTypes() {
+         AppDomain currentDomain = AppDomain.CurrentDomain;
+         currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
+         loadAssembly(this.GetType().Assembly.FullName);
          var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         foreach (string assembly_name in assemblies) {
-            bts.Log.Gen.info($"PrimitiveManager._load_all_streams assemblyName: {assembly_name}");
-            string resolved_path_name = assembly_name;
-            if (!System.IO.Path.IsPathRooted(assembly_name)) {
-               resolved_path_name = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assembly_name));
+         foreach (string assemblyName in assemblies) {
+            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
+            string resolvedAssemblyPath = assemblyName;
+            if (!System.IO.Path.IsPathRooted(assemblyName)) {
+               resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
             }
-            load_streams_from_assembly(resolved_path_name);
+            loadAssembly(resolvedAssemblyPath);
          }
       }
 
-      public Type get_dsl_op_type(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            return null;
-         return rv;
+      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
+         foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
+            if (args.Name.Equals(a.FullName))
+               return a;
+         return null;
       }
 
-      public void load_streams_from_assembly(string assembly_name) {
-         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assembly_name}");
-         if (_loaded_assemblies.Contains(assembly_name))
+      /// <summary>
+      /// Loads all the Stream types associated with the assembly
+      /// </summary>
+      /// <param name="assemblyName"></param>
+      public void loadAssembly(string assemblyName) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
+         if (myAssemblyNames.Contains(assemblyName))
             return;
-         _loaded_assemblies.Add(assembly_name);
-         Assembly loaded_assembly = GetType().Assembly.FullName.Equals(assembly_name) ?
+         myAssemblyNames.Add(assemblyName);
+         Assembly loadedAssembly = GetType().Assembly.FullName.Equals(assemblyName) ?
             GetType().Assembly :
-            Assembly.LoadFrom(assembly_name);
+            Assembly.LoadFrom(assemblyName);
 
-         foreach (Type type in loaded_assembly.GetTypes())
-            if (_is_loadable_stream(type)) {
-               bool is_visible = true;
-               string dsl_name = type.Name; //createDefaultName(type.Name);
+         foreach (Type type in loadedAssembly.GetTypes())
+            if (!type.IsInterface
+               && !type.IsAbstract
+               && type.IsSubclassOf(typeof(ViewStream))
+               && type.IsPublic) {
+               bool isVisible = true;
+               string userRep = createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
                   if (att is DataStreamUserRespresentation) {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
-                     is_visible = dsatt.isVisible;
-                     dsl_name = is_visible ?
-                        dsatt.friendlyName == null ? dsl_name : dsatt.friendlyName
+                     isVisible = dsatt.isVisible;
+                     userRep = isVisible ?
+                        dsatt.friendlyName == null ? userRep : dsatt.friendlyName
                         : null;
                      break;
                   }
-               if (is_visible) {
-                  // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
-                  _dslopname_to_type_map[dsl_name] = type;
+               if (isVisible) {
+                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  myTypeLookupTable[userRep] = type;
                }
             }
       }
 
+      public static string createDefaultName(string typeName) {
+         if (!typeName.EndsWith("Stream"))
+            return typeName;
+         string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
+         string startString = new String(new char[] { retVal[0] }).ToLower();
+         retVal = retVal.Remove(0, 1);
+         retVal = retVal.Insert(0, startString);
+         return retVal;
+      }
+
+
 
       public static PrimitiveManager instance {
-         get { return _self = _self == null ? new PrimitiveManager() : _self; }
+         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
       }
 
       public static void SetMarketDataClient(object value) {
@@ -96,20 +96,31 @@ namespace BCG {
          mdStreamFactory = new(value);
       }
 
-      public string dsl_to_type_name(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
-         return rv.Name;
+      /// <summary>
+      /// Returns the Type name corresponding to the primitive name
+      /// </summary>
+      /// <param name="friendlyName"></param>
+      /// <returns></returns>
+      public string find(string friendlyName) {
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         return (myTypeLookupTable[friendlyName] as Type).Name;
       }
 
+      /// <summary>
+      /// Constructs and returns a ViewStream of type corresponding to the friendly name
+      /// </summary>
+      /// <param name="friendlyName">friendly primitive name </param>
+      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
+      /// is the desired one</param>
+      /// <returns></returns>
       public ViewStream build(string friendlyName, ICollection args) {
          object[] constructorArgs = null;
          if (args != null && args.Count > 0) {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
-         var type = _dslopname_to_type_map[friendlyName] as Type
+         var type = myTypeLookupTable[friendlyName] as Type
             ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
             if (mdStreamFactory is null) {
@@ -124,14 +135,30 @@ namespace BCG {
             Activator.CreateInstance(type, constructorArgs) as ViewStream;
       }
 
-      public string[] get_dsl_op_names() {
-         return _dslopname_to_type_map.Keys.ToArray();
+      /// <summary>
+      /// Returns the name of all the primitives 
+      /// </summary>
+      /// <returns></returns>
+      public string[] getPrimitiveNames() {
+         string[] retVal = new string[myTypeLookupTable.Count];
+         myTypeLookupTable.Keys.CopyTo(retVal, 0);
+         return retVal;
       }
 
+      /// <summary>
+      /// Return the types of all the streams
+      /// </summary>
+      /// <returns></returns>
+      public Type[] getStreamTypes() {
+         Type[] retVal = new Type[myTypeLookupTable.Count];
+         myTypeLookupTable.Values.CopyTo(retVal, 0);
+         return retVal;
+      }
 
-      public ICollection get_dsl_op_cons_args(string dsl_op_name) {
+      public ICollection getConstructorSigns(string primitiveName) {
          ArrayList retVal = new ArrayList();
-         Type t = get_dsl_op_type(dsl_op_name);
+         Log.assert(myTypeLookupTable[primitiveName] != null);
+         Type t = myTypeLookupTable[primitiveName] as Type;
          foreach (ConstructorInfo ci in t.GetConstructors()) {
             string sign = "";
             sign = t.Name + " (";
@@ -145,18 +172,9 @@ namespace BCG {
          return retVal;
       }
 
-      public string get_dslop_assembly_name(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
-         return rv.Assembly.FullName;
+      public string getAssemblyName(string primitiveName) {
+         Log.assert(myTypeLookupTable[primitiveName] != null);
+         return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
       }
-
-      public string get_qualified_op_name(string dsl_op_name) {
-         Type dsl_op_type = get_dsl_op_type(dsl_op_name);
-         Log.assert(dsl_op_type != null);
-         return $"{dsl_op_type.Namespace}.{dsl_op_name}";
-      }
-
    }
 }

src/DataStreamSuite/DataStreamCore/IDataPacket.cs

@@ -13,6 +13,7 @@ namespace BCG {
       IBlueRowCollection newRows { get; }
       IBlueRowCollection addedRows { get; }
       IBlueRowCollection deletedRows { get; }
+      IReadOnlyDictionary<string, object> Metadata { get; }
       //this is an ICollection of IDictionary
       void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
    }

src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs

@@ -3,23 +3,23 @@ using System.IO;
 using System.Collections.Generic;
 using System.Linq;
 
-using BlueOrb.Data;
 using bts;
-using System.Threading;
 
-namespace BCG {
-   // This class is responsible for mantaining the state for a table. It gets the
-   // bytes from the pubsubserver via the handle_updates call 
-   // (called by VSD, - VSD is polling the SubscriberCOre for udpates), deserializes,
+namespace BCG
+{
+   // This class is responsible for mantaining the state fo a table. It gets the
+   // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
    // updates the state of the table, and fires the TableUpdateDelegate.
-   public class TableAccumulator2 : ITableAccumulator {
+   public class TableAccumulator2 : ITableAccumulator
+   {
       private readonly string _tag;
       private readonly ISubscriberCore _subscriber_core;
       IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
       private BlueTable _table;
       private TableUpdateDelegate _on_update_delegate;
 
-      public TableAccumulator2(string tag, ISubscriberCore sub_core) {
+      public TableAccumulator2(string tag, ISubscriberCore sub_core)
+      {
          _tag = tag;
          _subscriber_core = sub_core;
       }
@@ -28,54 +28,61 @@ namespace BCG {
 
       //public bool is_active() { return _on_update_delegate != null; }
 
-      public event TableUpdateDelegate on_update {
-         add {
+      public event TableUpdateDelegate on_update
+      {
+         add
+         {
             Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
-            if (_on_update_delegate == null) {
+            if (_on_update_delegate == null)
+            {
                //ie first subscription
                Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
                _subscriber_core.subscribe(_tag);
             }
             _on_update_delegate =
                Delegate.Combine(_on_update_delegate, value) as TableUpdateDelegate;
-            // a new subsriber should get a full callback immediatly (new rows, 0 deleted, all added)
             if (_table != null)
                value(_table, _table, new BlueTable(_table.schema));
          }
-         remove {
+         remove
+         {
             Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
-            if (_on_update_delegate != null) {
+            if (_on_update_delegate != null)
+            {
                _on_update_delegate =
                   Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
                int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
                Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
-               if (_on_update_delegate == null) {
+               if (_on_update_delegate == null)
+               {
                   _subscriber_core.unsubscribe(_tag);
-                  _table = null;
+                  // TODO
+                  //_table = null;
                }
             }
 
          }
       }
       private int _update_no = 0;
-      public void handle_updates(IEnumerable<byte[]> msgs) {
+      public void handle_updates(IEnumerable<byte[]> msgs)
+      {
          if (msgs == null || msgs.Count() == 0)
             return;
          Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
          IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
-         int i = 0;
-         foreach (var msg in msgs) {
-            bts.StreamIOMessage new_msg_txt = _deser.deserialize(msg);
+         foreach (var msg in msgs)
+         {
+            bts.StreamIOMessage str_msg = _deser.deserialize(msg);
             var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
-            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} object_hash: {GetHashCode()} thread_id: {Thread.CurrentThread.ManagedThreadId} msg: {i++ / msgs.Count()} msg_sz: {msg.Length} new_msg_txt: {new_msg_txt} pre_update_count: {state_txt}");
-
-            switch (new_msg_txt.op) {
+            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
+            switch (str_msg.op)
+            {
                case bts.StreamIOMessage.StreamOp.CREATE:
-                  _table = new BlueTable(new_msg_txt.create_message.rows);
+                  _table = new BlueTable(str_msg.create_message.rows);
                   break;
                case bts.StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
-                  _table.deleteRows(new_msg_txt.multiple_remove_add_message.removed_rows);
-                  _table.addRows(new_msg_txt.multiple_remove_add_message.added_rows);
+                  _table.deleteRows(str_msg.multiple_remove_add_message.removed_rows);
+                  _table.addRows(str_msg.multiple_remove_add_message.added_rows);
                   break;
             }
          }
@@ -84,16 +91,11 @@ namespace BCG {
          IBlueRowCollection d_rows, a_rows;
          //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
          BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
-         Log.Gen.info($"TableAccumulator2.handle_updates firing update, _table: {_table.rows_id_string()} d_rows: {d_rows.rows_id_string()} a_rows: {a_rows.rows_id_string()}");
-         if (a_rows.Count == 0 && d_rows.Count == 0 && table_0 != null) {
-            Log.Gen.info($"TableAccmulator2.handle_updates ex! _table: {_table.ToDebugString()} table_0: {table_0.ToDebugString()} a_rows: {a_rows.ToDebugString()} d_rows: {d_rows.ToDebugString()}");
-            //same table was pubslished (maybe surpress it on the publisher or pubsubs)
-            return;
-         }
          _on_update_delegate?.Invoke(_table, a_rows, d_rows);
       }
 
-      public void update(Stream[] streams) {
+      public void update(Stream[] streams)
+      {
       }
 
       #endregion

src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs

@@ -30,7 +30,7 @@ namespace bts {
       }
    }
 
-   // This class is created by the SingleTagData for stream tag_type. Every DS tag has a PubsubServerSingleStreamTagHandler
+   // This class is created by the SingleTagData for stream tag_type. Every pub
 
    public class PubsubServerSingleStreamTagHandler : IPubSubServerTagHandler {
       // this is kept around to deserielize the incoming messages and keep the state of the table

src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs

@@ -117,6 +117,9 @@ namespace BCG {
                      addWorker(op.w);
                   else
                      removeWorker(op.w);
+            }
+            catch (InvalidOperationException ex) {
+               bts.Log.Gen.error($"ViewStreamDatabase worker collection was modified during enumeration: {ex}");
             } finally {
                _pending_worker_ops.Clear();
                _is_worker_call_in_progress = false;

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
    }
 
    public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.dsl_to_type_name(myName) + " (";
+      string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
 
       return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
    }

src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs

@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
 
+using global::Spark.Core.Disposables;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
 using System.Linq;

src/DataStreamSuite/DataStreamPublisher/WSPublishers/DatabentoPublisher.py

@@ -1,146 +0,0 @@
-"""
-Requires:
-   pip install -U databento
-   pip install websockets
-
-Sources:
-   https://databento.com/docs/quickstart/set-up
-   https://databento.com/docs/schemas-and-data-formats/bbo
-"""
-
-import datetime as dt
-import subprocess
-import sys
-import time
-
-try:
-   import databento as db
-except ImportError:
-   print("Installing missing dependency: databento")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
-   import databento as db
-
-from databento.common.error import BentoError
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-from databento_transformer import DatabentoTransformer
-from pubsub_client import PubSubClient
-
-
-API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
-DATASET = "GLBX.MDP3"
-SYMBOL = "ES.FUT"
-PUBSUB_WS_URL = "ws://localhost:19092/ws/"
-BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
-TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
-DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
-
-_pubsub_client = PubSubClient(PUBSUB_WS_URL)
-_transformer = DatabentoTransformer(
-   symbol=SYMBOL,
-   bbo_table_name=BBO_TABLE_NAME,
-   trades_table_name=TRADES_TABLE_NAME,
-   definitions_table_name=DEFINITIONS_TABLE_NAME,
-)
-
-
-def _publish_contract_definitions() -> None:
-   historical_client = db.Historical(API_KEY)
-   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
-   schema_range = dataset_range.get("schema", {}).get("definition", {})
-   available_end = schema_range.get("end", dataset_range["end"])
-
-   if isinstance(available_end, str):
-      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
-   else:
-      available_end_dt = available_end
-
-   # Databento's range end is exclusive. Back off slightly so the request stays
-   # strictly inside the available window.
-   end_dt = available_end_dt - dt.timedelta(seconds=1)
-   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
-   start = start_dt.isoformat()
-   end = end_dt.isoformat()
-
-   definition_data = historical_client.timeseries.get_range(
-      dataset=DATASET,
-      schema="definition",
-      stype_in="parent",
-      symbols=SYMBOL,
-      start=start,
-      end=end,
-   )
-
-   df = definition_data.to_df()
-   payload = _transformer.build_definitions_payload(df)
-   if payload is None:
-      print("No Databento contract definitions returned")
-      return
-
-   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
-   _pubsub_client.publish(payload)
-
-
-def on_record(record) -> None:
-   # Print every callback first so we can see whether Databento is flowing
-   # even if our row extraction logic misses a field shape.
-   print(record)
-
-   payloads = _transformer.transform_record(record)
-   if payloads:
-      for payload in payloads:
-         _pubsub_client.publish(payload)
-      return
-
-   print("Skipping record with no BBO/trade fields")
-
-
-def main() -> None:
-   _pubsub_client.start()
-   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
-   try:
-      _publish_contract_definitions()
-   except Exception as ex:
-      print(f"Contract definition publish failed: {ex}", flush=True)
-
-   while True:
-      try:
-         client = db.Live(key=API_KEY)
-
-         client.subscribe(
-            dataset=DATASET,
-            schema="bbo-1s",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-         client.subscribe(
-            dataset=DATASET,
-            schema="trades",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-
-         client.add_callback(on_record)
-         client.start()
-         print("Databento client started, waiting for BBO and trade records...", flush=True)
-         client.block_for_close()
-         print("Databento client closed cleanly", flush=True)
-         break
-      except BentoError as ex:
-         print(f"Databento live connection error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-      except Exception as ex:
-         print(f"Databento publisher unexpected error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-
-
-if __name__ == "__main__":
-   main()

src/DataStreamSuite/DataStreamPublisher/WSPublishers/databento_transformer.py

@@ -1,244 +0,0 @@
-import datetime as dt
-
-
-class DatabentoTransformer:
-   def __init__(
-      self,
-      symbol: str,
-      bbo_table_name: str,
-      trades_table_name: str,
-      definitions_table_name: str,
-   ):
-      self._symbol = symbol
-      self._bbo_table_name = bbo_table_name
-      self._trades_table_name = trades_table_name
-      self._definitions_table_name = definitions_table_name
-      self._instrument_id_to_symbol: dict[int, str] = {}
-      self._last_bbo_row: dict | None = None
-      self._trade_row_id = 0
-      self._trade_table_created = False
-
-   def transform_record(self, record) -> list[dict]:
-      payload = self._handle_bbo_record(record)
-      if payload is not None:
-         return [payload]
-
-      payload = self._handle_trade_record(record)
-      if payload is not None:
-         return [payload]
-
-      return []
-
-   def build_definitions_payload(self, definition_df) -> dict | None:
-      if definition_df is None or definition_df.empty:
-         return None
-
-      df = definition_df.reset_index()
-      if "instrument_class" in df.columns:
-         df = df[df["instrument_class"].astype(str) == "F"]
-      if "security_type" in df.columns:
-         df = df[df["security_type"].astype(str) == "FUT"]
-      if "raw_symbol" in df.columns:
-         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
-         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
-
-      rows: list[dict] = []
-      for idx, row in enumerate(df.itertuples(index=False), start=1):
-         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
-         security_name = str(getattr(row, "raw_symbol", ""))
-         if instrument_id > 0 and security_name:
-            self._instrument_id_to_symbol[instrument_id] = security_name
-
-         rows.append({
-            "id": idx,
-            "security_name": security_name,
-            "instrument_id": instrument_id,
-            "instrument_class": str(getattr(row, "instrument_class", "")),
-            "security_type": str(getattr(row, "security_type", "")),
-            "asset": str(getattr(row, "asset", "")),
-            "exchange": str(getattr(row, "exchange", "")),
-            "currency": str(getattr(row, "currency", "")),
-            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
-            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
-            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
-            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
-            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
-            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
-            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
-            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
-            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
-            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
-            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
-            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
-            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
-            "underlying": str(getattr(row, "underlying", "")),
-         })
-
-      if not rows:
-         return None
-
-      return {
-         "op": "create_table",
-         "table_name": self._definitions_table_name,
-         "rows": rows,
-      }
-
-   def _handle_bbo_record(self, record) -> dict | None:
-      row = self._to_bbo_row(record)
-      if row is None:
-         return None
-
-      if self._last_bbo_row is None:
-         payload = {
-            "op": "create_table",
-            "table_name": self._bbo_table_name,
-            "rows": [row],
-         }
-      else:
-         payload = {
-            "op": "multiple_remove_add",
-            "table_name": self._bbo_table_name,
-            "removed_rows": [self._last_bbo_row],
-            "added_rows": [row],
-         }
-
-      self._last_bbo_row = row
-      return payload
-
-   def _handle_trade_record(self, record) -> dict | None:
-      row = self._to_trade_row(record)
-      if row is None:
-         return None
-
-      row["id"] = self._trade_row_id
-      self._trade_row_id += 1
-
-      if not self._trade_table_created:
-         payload = {
-            "op": "create_table",
-            "table_name": self._trades_table_name,
-            "rows": [row],
-         }
-         self._trade_table_created = True
-         return payload
-
-      return {
-         "op": "multiple_remove_add",
-         "table_name": self._trades_table_name,
-         "removed_rows": [],
-         "added_rows": [row],
-      }
-
-   def _to_bbo_row(self, record) -> dict | None:
-      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
-      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
-      if bid_px is None and ask_px is None:
-         return None
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "bid_px": float(bid_px or 0.0),
-         "ask_px": float(ask_px or 0.0),
-         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
-         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
-      }
-
-   def _to_trade_row(self, record) -> dict | None:
-      price = self._get_attr(record, "price", "px")
-      size = self._get_attr(record, "size")
-      if price is None and size is None:
-         return None
-
-      side = self._get_attr(record, "side", "aggressor_side", default="")
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "price": float(price or 0.0),
-         "size": int(size or 0),
-         "side": str(side),
-      }
-
-   def _get_security_name(self, record) -> str:
-      instrument_id = self._get_attr(record, "instrument_id", default=0)
-      try:
-         instrument_id_int = int(instrument_id or 0)
-      except (TypeError, ValueError):
-         instrument_id_int = 0
-
-      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
-         return self._instrument_id_to_symbol[instrument_id_int]
-
-      value = self._get_attr(
-         record,
-         "symbol",
-         "raw_symbol",
-         "stype_out_symbol",
-         "stype_in_symbol",
-         default=self._symbol,
-      )
-      return str(value)
-
-   def _get_timestamp(self, record) -> int:
-      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
-      try:
-         return int(value)
-      except (TypeError, ValueError):
-         return 0
-
-   def _get_attr(self, record, *names, default=None):
-      for name in names:
-         if hasattr(record, name):
-            value = getattr(record, name)
-            if value is not None:
-               return value
-      return default
-
-   def _get_level_attr(self, record, name, default=None):
-      levels = self._get_attr(record, "levels", default=None)
-      if levels:
-         level0 = levels[0]
-         if hasattr(level0, name):
-            value = getattr(level0, name)
-            if value is not None:
-               return value
-      return default
-
-   def _ns_to_iso(self, timestamp_ns) -> str:
-      if timestamp_ns is None:
-         return ""
-
-      if hasattr(timestamp_ns, "isoformat"):
-         try:
-            if hasattr(timestamp_ns, "to_pydatetime"):
-               timestamp_dt = timestamp_ns.to_pydatetime()
-            else:
-               timestamp_dt = timestamp_ns
-
-            if timestamp_dt is None:
-               return ""
-
-            if getattr(timestamp_dt, "tzinfo", None) is None:
-               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
-            return timestamp_dt.isoformat()
-         except (TypeError, ValueError, AttributeError):
-            pass
-
-      try:
-         ts_int = int(timestamp_ns)
-         if ts_int <= 0:
-            return ""
-         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
-      except (TypeError, ValueError, OSError, OverflowError):
-         return ""
-
-   def _nano_to_float(self, value) -> float:
-      try:
-         return float(value) / 1_000_000_000.0
-      except (TypeError, ValueError):
-         return 0.0

src/DataStreamSuite/DataStreamPublisher/WSPublishers/pubsub_client.py

@@ -1,58 +0,0 @@
-import asyncio
-import json
-import queue
-import threading
-
-import websockets
-
-
-class PubSubClient:
-   def __init__(self, ws_url: str):
-      self._ws_url = ws_url
-      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
-      self._sender_thread: threading.Thread | None = None
-
-   def start(self) -> threading.Thread:
-      if self._sender_thread is None or not self._sender_thread.is_alive():
-         self._sender_thread = self._start_sender_thread()
-      return self._sender_thread
-
-   def publish(self, payload: dict) -> None:
-      self._outgoing_queue.put(payload)
-
-   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
-      self.publish({
-         "op": "create_table",
-         "table_name": table_name,
-         "rows": rows,
-      })
-
-   def publish_multiple_remove_add(
-      self,
-      table_name: str,
-      removed_rows: list[dict],
-      added_rows: list[dict],
-   ) -> None:
-      self.publish({
-         "op": "multiple_remove_add",
-         "table_name": table_name,
-         "removed_rows": removed_rows,
-         "added_rows": added_rows,
-      })
-
-   def _start_sender_thread(self) -> threading.Thread:
-      thread = threading.Thread(target=self._run_send_loop, daemon=True)
-      thread.start()
-      return thread
-
-   def _run_send_loop(self) -> None:
-      asyncio.run(self._ws_send_loop())
-
-   async def _ws_send_loop(self) -> None:
-      async with websockets.connect(self._ws_url) as ws:
-         print(f"Connected to PubSubServer websocket: {self._ws_url}")
-         while True:
-            payload = await asyncio.to_thread(self._outgoing_queue.get)
-            body = json.dumps(payload)
-            print(body)
-            await ws.send(body)

src/DataStreamSuite/DataStreamPublisher/appsettings.json

@@ -59,9 +59,14 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
         },
         {
           "TagType": ".*",
@@ -85,4 +90,4 @@
       }
     ]
   }
-}
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -21,55 +21,11 @@ using System.Reactive.Subjects;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Windows.Forms;
-using BlueOrb.Pubsub.Diagnostics;
 
 namespace BCG
 {
    public partial class MainForm : System.Windows.Forms.Form
    {
-      private const string LastQueryRegistryKey = "LastQuery";
-      private const string RecentSavedFilesRegistryKey = "RecentSavedFiles";
-      private const string DateTimeDisplayFormat = "M/d/yyyy h:mm:ss.fff tt";
-      private const int MaxRecentSavedFiles = 10;
-      private static readonly Color QueryEditorDefaultColor = Color.White;
-      private static readonly Color QueryEditorCommentColor = Color.FromArgb(106, 115, 125);
-      private static readonly Color QueryEditorOperatorColor = Color.FromArgb(86, 156, 214);
-      private static readonly Color QueryEditorAssignmentNameColor = Color.FromArgb(78, 201, 176);
-      private static readonly Color GridBackgroundColor = Color.FromArgb(37, 37, 38);
-      private static readonly Color GridAltRowColor = Color.FromArgb(43, 43, 48);
-      private static readonly Color GridHeaderColor = Color.FromArgb(62, 62, 66);
-      private static readonly Color GridGridLineColor = Color.FromArgb(58, 58, 61);
-      private static readonly Color GridSelectionColor = Color.FromArgb(9, 71, 113);
-      private static readonly Color GridSelectionForeColor = Color.White;
-      private static readonly Color GridTimestampColor = Color.FromArgb(206, 215, 222);
-      private static readonly Color GridIdentifierColor = Color.FromArgb(156, 163, 175);
-      private static readonly Color GridIndicatorColor = Color.FromArgb(156, 220, 254);
-
-      class PrimitiveTreeNodeComparer : IComparer
-      {
-         public int Compare(object? x, object? y)
-         {
-            var left = x as TreeNode;
-            var right = y as TreeNode;
-            if (left is null || right is null)
-               return 0;
-
-            string left_key = _sort_key(left);
-            string right_key = _sort_key(right);
-            return StringComparer.OrdinalIgnoreCase.Compare(left_key, right_key);
-         }
-
-         static string _sort_key(TreeNode node)
-         {
-            if (node.Parent is null)
-               return node.Text;
-
-            if (node.Tag is string primitive_name)
-               return primitive_name;
-
-            return node.Text;
-         }
-      }
       // static async Task _run() {
       //    var login = "bts";
       //    var passwd = String.Empty;
@@ -84,8 +40,6 @@ namespace BCG
       private string? myLastFileName;
       private ViewStream? myCurrentStream;
       private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
-      private readonly List<string> _recent_saved_files = new List<string>();
-      private ToolStripSeparator? _recent_saved_files_separator;
       // private string mySite = SitesCore.getMySiteName ();
       private string mySite = "TODO";
       private string myLastClassName = "MyView";
@@ -95,20 +49,11 @@ namespace BCG
       private TextWriter? myConsoleOutput;
       IDataPacket? pendingData;
       IDisposable? titleSub;
-      readonly ToolTip _primitive_tool_tip = new ToolTip();
-      readonly Dictionary<string, string?> _primitive_source_path_cache = new Dictionary<string, string?>();
-      Dictionary<string, string?>? _source_file_index;
-      string? _repo_root_path;
-      char? _last_primitive_search_char;
-      TreeNode? _last_primitive_search_assembly_node;
-      bool _is_applying_query_syntax_highlighting;
-      HashSet<string>? _dsl_operator_names;
 
       public MainForm()
       {
          //Task.Run(async () => await _run());
          InitializeComponent();
-         _apply_grid_theme();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
          myKQ = "TODO";
          var handleCreatedSubject = new Subject<bool>();
@@ -127,12 +72,13 @@ namespace BCG
          myWindowState = new PersistWindowState();
          myWindowState.parent = this;
          myWindowState.saveState = false;
-         myWindowState.LoadState += myLoadPersistedState;
-         myWindowState.SaveState += mySavePersistedState;
 
          this.Load += new EventHandler(myLoad);
          this.Closing += new CancelEventHandler(myClosing);
-         myQueryTextBox.TextChanged += (o, e) => _apply_query_syntax_highlighting();
+         myQueryTextBox.TextChanged += (o, e) =>
+         {
+            myQueryTextBox.ForeColor = Color.White;
+         };
       }
 
       event Action? titleInvalidated;
@@ -140,20 +86,14 @@ namespace BCG
       public string query_text
       {
          get { return myQueryTextBox.Text; }
-         set
-         {
-            myQueryTextBox.Text = value;
-            _apply_query_syntax_highlighting();
-         }
+         set { myQueryTextBox.Text = value; }
       }
 
       /// <summary>
       /// Clean up any resources being used.
       /// </summary>
-      protected override void Dispose(bool disposing)
-      {
-         if (disposing)
-         {
+      protected override void Dispose(bool disposing) {
+         if (disposing) {
             Disposable.Dispose(ref titleSub);
             components?.Dispose();
          }
@@ -164,7 +104,7 @@ namespace BCG
       {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
          //ViewStream vs = new PubSubServerStateStream();
-         ViewStream vs = new PubSubServerStateStream(ViewStreamDatabase.instance);
+         ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
          return new FilterStream(vs, "tag~='DS_'");
          //return new LiteralStream("tag/String", "DS_table1;DS_table2");
       }
@@ -174,8 +114,7 @@ namespace BCG
          titleInvalidated?.Invoke();
       }
 
-      void refreshTitle(IBlueRowCollection rows)
-      {
+      void refreshTitle(IBlueRowCollection rows) {
          string route_txt = String.Empty;
          foreach (IBlueRow r in rows)
          {
@@ -196,11 +135,11 @@ namespace BCG
       private IDictionary myGetLibraryData()
       {
          SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.get_dsl_op_names();
+         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
          foreach (string pName in primitiveNames)
          {
-            string aName = PrimitiveManager.instance.get_dslop_assembly_name(pName);
-            ICollection cNames = PrimitiveManager.instance.get_dsl_op_cons_args(pName);
+            string aName = PrimitiveManager.instance.getAssemblyName(pName);
+            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
             if (!retVal.Contains(aName))
                retVal.Add(aName, new SortedList());
             (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
@@ -212,65 +151,6 @@ namespace BCG
          return retVal;
       }
 
-      HashSet<string> _get_dsl_operator_names()
-      {
-         if (_dsl_operator_names != null)
-            return _dsl_operator_names;
-
-         _dsl_operator_names = new HashSet<string>(
-            PrimitiveManager.instance.get_dsl_op_names().Cast<string>(),
-            StringComparer.OrdinalIgnoreCase);
-         return _dsl_operator_names;
-      }
-
-      void _apply_query_syntax_highlighting()
-      {
-         if (_is_applying_query_syntax_highlighting || myQueryTextBox.IsDisposed)
-            return;
-
-         _is_applying_query_syntax_highlighting = true;
-         try
-         {
-            int selection_start = myQueryTextBox.SelectionStart;
-            int selection_length = myQueryTextBox.SelectionLength;
-
-            myQueryTextBox.SuspendLayout();
-            myQueryTextBox.SelectAll();
-            myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"^\s*([A-Za-z_]\w*)\s*=", RegexOptions.Multiline))
-            {
-               Group group = match.Groups[1];
-               myQueryTextBox.Select(group.Index, group.Length);
-               myQueryTextBox.SelectionColor = QueryEditorAssignmentNameColor;
-            }
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"\b([A-Za-z_]\w*)\s*\("))
-            {
-               Group group = match.Groups[1];
-               if (!_get_dsl_operator_names().Contains(group.Value))
-                  continue;
-               myQueryTextBox.Select(group.Index, group.Length);
-               myQueryTextBox.SelectionColor = QueryEditorOperatorColor;
-            }
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"//.*$", RegexOptions.Multiline))
-            {
-               myQueryTextBox.Select(match.Index, match.Length);
-               myQueryTextBox.SelectionColor = QueryEditorCommentColor;
-            }
-
-            myQueryTextBox.Select(selection_start, selection_length);
-            if (selection_length == 0)
-               myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
-            myQueryTextBox.ResumeLayout();
-         }
-         finally
-         {
-            _is_applying_query_syntax_highlighting = false;
-         }
-      }
-
 
       private void myLoadAssemblies()
       {
@@ -281,18 +161,13 @@ namespace BCG
             TreeNode assemblyNode = new TreeNode(de.Key as String);
             foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
             {
-               string primitive_name = innerDe.Key as string;
-               TreeNode primitiveNode = new TreeNode(primitive_name)
-               {
-                  Tag = primitive_name
-               };
+               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
                foreach (string con in (innerDe.Value as ICollection))
                   primitiveNode.Nodes.Add(con);
                assemblyNode.Nodes.Add(primitiveNode);
             }
             myTreeView.Nodes.Add(assemblyNode);
          }
-         myTreeView.Sort();
          /*
          myPrimitivesTextBox.Clear ();
          ArrayList al = new ArrayList(PrimitiveManager.instance.getPrimitiveNames());
@@ -307,8 +182,7 @@ namespace BCG
 
       private void myHandleTableListChange(object o)
       {
-         BeginInvoke(() =>
-         {
+         BeginInvoke(() => {
             tagRows = ((IDataPacket)o).newRows;
             myUpdateTableList();
          });
@@ -321,19 +195,6 @@ namespace BCG
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
-
-      private void myLoadPersistedState(object sender, BlueRegistry blueReg)
-      {
-         myQueryTextBox.Text = blueReg.getValue(RegistryRoot.HKCU, LastQueryRegistryKey, string.Empty)?.ToString() ?? string.Empty;
-         _load_recent_saved_files(blueReg.getValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Empty)?.ToString());
-         _refresh_recent_saved_files_menu();
-      }
-
-      private void mySavePersistedState(object sender, BlueRegistry blueReg)
-      {
-         blueReg.setValue(RegistryRoot.HKCU, LastQueryRegistryKey, myQueryTextBox.Text ?? string.Empty);
-         blueReg.setValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Join("|", _recent_saved_files));
-      }
       #endregion
 
       private Stopwatch _stopwatch = new Stopwatch();
@@ -426,7 +287,6 @@ namespace BCG
             }
             myLastFileName = fileName;
             this.Text = fileName;
-            _track_recent_saved_file(fileName);
          }
          catch (Exception exc)
          {
@@ -511,122 +371,6 @@ namespace BCG
          }
       }
 
-      private void myApplyDateTimeFormatting(IBlueTableSchema schema)
-      {
-         var dateTimeColumns = new HashSet<string>(
-            schema.columnInfo
-               .Where(ci => ci.colType == typeof(DateTime))
-               .Select(ci => ci.colName));
-
-         foreach (DataGridViewColumn column in myDataGrid.Columns)
-         {
-            if (!dateTimeColumns.Contains(column.DataPropertyName) && !dateTimeColumns.Contains(column.Name))
-               continue;
-
-            column.DefaultCellStyle.Format = DateTimeDisplayFormat;
-            column.Width = Math.Max(column.Width, 220);
-         }
-      }
-
-      void _apply_grid_theme()
-      {
-         myDataGrid.BackgroundColor = GridBackgroundColor;
-         myDataGrid.BorderStyle = BorderStyle.None;
-         myDataGrid.EnableHeadersVisualStyles = false;
-         myDataGrid.GridColor = GridGridLineColor;
-         myDataGrid.CellBorderStyle = DataGridViewCellBorderStyle.SingleHorizontal;
-         myDataGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect;
-         myDataGrid.MultiSelect = false;
-         myDataGrid.RowHeadersVisible = false;
-         myDataGrid.ColumnHeadersBorderStyle = DataGridViewHeaderBorderStyle.Single;
-         myDataGrid.RowHeadersBorderStyle = DataGridViewHeaderBorderStyle.None;
-         myDataGrid.ColumnHeadersHeight = 36;
-         myDataGrid.RowTemplate.Height = 24;
-         myDataGrid.Font = new Font("Consolas", 9F, FontStyle.Regular, GraphicsUnit.Point);
-
-         myDataGrid.DefaultCellStyle.BackColor = GridBackgroundColor;
-         myDataGrid.DefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.DefaultCellStyle.SelectionBackColor = GridSelectionColor;
-         myDataGrid.DefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
-         myDataGrid.DefaultCellStyle.Padding = new Padding(4, 1, 4, 1);
-
-         myDataGrid.AlternatingRowsDefaultCellStyle.BackColor = GridAltRowColor;
-         myDataGrid.AlternatingRowsDefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionBackColor = GridSelectionColor;
-         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
-
-         myDataGrid.ColumnHeadersDefaultCellStyle.BackColor = GridHeaderColor;
-         myDataGrid.ColumnHeadersDefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionBackColor = GridHeaderColor;
-         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionForeColor = Color.White;
-         myDataGrid.ColumnHeadersDefaultCellStyle.Font = new Font("Consolas", 9F, FontStyle.Bold, GraphicsUnit.Point);
-         myDataGrid.ColumnHeadersDefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
-         myDataGrid.ColumnHeadersDefaultCellStyle.Padding = new Padding(6, 2, 6, 2);
-      }
-
-      void _apply_grid_column_styling(IBlueTableSchema schema)
-      {
-         var column_types = schema.columnInfo.ToDictionary(ci => ci.colName, ci => ci.colType, StringComparer.OrdinalIgnoreCase);
-         foreach (DataGridViewColumn column in myDataGrid.Columns)
-         {
-            if (!column_types.TryGetValue(column.DataPropertyName, out var col_type) &&
-                !column_types.TryGetValue(column.Name, out col_type))
-            {
-               continue;
-            }
-
-            column.SortMode = DataGridViewColumnSortMode.NotSortable;
-
-            if (_is_numeric_type(col_type))
-            {
-               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleRight;
-            }
-            else
-            {
-               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
-            }
-
-            if (col_type == typeof(DateTime))
-            {
-               column.DefaultCellStyle.ForeColor = GridTimestampColor;
-            }
-            else if (string.Equals(column.HeaderText, "id", StringComparison.OrdinalIgnoreCase))
-            {
-               column.DefaultCellStyle.ForeColor = GridIdentifierColor;
-            }
-            else if (_looks_like_indicator(column.HeaderText))
-            {
-               column.DefaultCellStyle.ForeColor = GridIndicatorColor;
-            }
-            else
-            {
-               column.DefaultCellStyle.ForeColor = Color.White;
-            }
-         }
-
-         static bool _is_numeric_type(Type type)
-         {
-            return type == typeof(int) ||
-                   type == typeof(long) ||
-                   type == typeof(float) ||
-                   type == typeof(double) ||
-                   type == typeof(decimal) ||
-                   type == typeof(short) ||
-                   type == typeof(uint) ||
-                   type == typeof(ulong);
-         }
-
-         static bool _looks_like_indicator(string column_name)
-         {
-            return column_name.Contains("rsi", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("macd", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("atr", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("moving_average", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("signal", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("histogram", StringComparison.OrdinalIgnoreCase);
-         }
-      }
-
       // this is the stream callback
       private void myCallback(object o)
       {
@@ -642,8 +386,7 @@ namespace BCG
       private void myTimer_Tick(object sender, System.EventArgs e)
       {
          var dp = Interlocked.Exchange(ref pendingData, null);
-         if (dp is null)
-         {
+         if (dp is null) {
             return;
          }
          _stopwatch.Stop();
@@ -660,10 +403,6 @@ namespace BCG
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
 
          myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
-         _apply_grid_theme();
-         myApplyDateTimeFormatting(dp.newRows.schema);
-         _apply_grid_column_styling(dp.newRows.schema);
-         myDataGrid.ClearSelection();
          var force = false;
          if (myChangeStyle && force)
          {
@@ -736,7 +475,7 @@ namespace BCG
       private void myMenuItemOpenClick(object sender, System.EventArgs e)
       {
          OpenFileDialog od = new OpenFileDialog();
-         od.Filter = "All files (*.*)|*.*|C# class (*.cs)|*.cs";
+         od.Filter = "C# class (*.cs)|*.cs";
 
          if (od.ShowDialog() == DialogResult.OK)
             loadFile(od.FileName);
@@ -748,7 +487,6 @@ namespace BCG
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
-         _track_recent_saved_file(fileName);
       }
 
       private void myMenuItemSaveClick(object sender, System.EventArgs e)
@@ -771,75 +509,6 @@ namespace BCG
             mySave(sd.FileName);
       }
 
-      private void _load_recent_saved_files(string? serialized_paths)
-      {
-         _recent_saved_files.Clear();
-         if (string.IsNullOrWhiteSpace(serialized_paths))
-            return;
-
-         foreach (string path in serialized_paths.Split('|'))
-         {
-            if (string.IsNullOrWhiteSpace(path))
-               continue;
-            if (_recent_saved_files.Contains(path, StringComparer.OrdinalIgnoreCase))
-               continue;
-            _recent_saved_files.Add(path);
-            if (_recent_saved_files.Count >= MaxRecentSavedFiles)
-               break;
-         }
-      }
-
-      private void _track_recent_saved_file(string file_name)
-      {
-         _recent_saved_files.RemoveAll(path => string.Equals(path, file_name, StringComparison.OrdinalIgnoreCase));
-         _recent_saved_files.Insert(0, file_name);
-         if (_recent_saved_files.Count > MaxRecentSavedFiles)
-            _recent_saved_files.RemoveRange(MaxRecentSavedFiles, _recent_saved_files.Count - MaxRecentSavedFiles);
-         _refresh_recent_saved_files_menu();
-      }
-
-      private void _refresh_recent_saved_files_menu()
-      {
-         var file_menu = myToolStripMenuItemFile.DropDownItems;
-         for (int i = file_menu.Count - 1; i >= 0; i--)
-         {
-            if (file_menu[i] is ToolStripMenuItem menu_item && Equals(menu_item.Tag, "recent-saved-file"))
-               file_menu.RemoveAt(i);
-         }
-
-         if (_recent_saved_files_separator != null)
-         {
-            file_menu.Remove(_recent_saved_files_separator);
-            _recent_saved_files_separator.Dispose();
-            _recent_saved_files_separator = null;
-         }
-
-         if (_recent_saved_files.Count == 0)
-            return;
-
-         int insert_index = file_menu.IndexOf(myToolStripMenuItemExit);
-         if (insert_index < 0)
-            insert_index = file_menu.Count;
-
-         _recent_saved_files_separator = new ToolStripSeparator();
-         file_menu.Insert(insert_index, _recent_saved_files_separator);
-         insert_index++;
-
-         foreach (string path in _recent_saved_files)
-         {
-            var menu_item = new ToolStripMenuItem(Path.GetFileName(path))
-            {
-               Tag = "recent-saved-file",
-               ToolTipText = path,
-               BackColor = Color.FromArgb(36, 36, 49),
-               ForeColor = Color.Lime
-            };
-            menu_item.Click += (_, __) => loadFile(path);
-            file_menu.Insert(insert_index, menu_item);
-            insert_index++;
-         }
-      }
-
       private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
       {
          if (myTableListBox.SelectedItem != null)
@@ -930,7 +599,6 @@ namespace BCG
          }
          Console.WriteLine("Redirecting console ouput to this window...");
 
-         myTreeView.TreeViewNodeSorter = new PrimitiveTreeNodeComparer();
          myLoadAssemblies();
 
          // ViewStreamDatabase.instance = new ViewStreamDatabase();
@@ -938,9 +606,6 @@ namespace BCG
          myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
          myTabControl.SelectedIndex = 3;
          myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
-         myTreeView.KeyPress += myHandleTreeViewKeyPress;
-         myTreeView.NodeMouseHover += myHandleTreeNodeMouseHover;
-         myTreeView.MouseLeave += myHandleTreeViewMouseLeave;
          //do this lazily so we don't have to have bps, etc up to run dsqa
          //_library = new LibraryForm(this);
       }
@@ -981,138 +646,12 @@ namespace BCG
          {
             if (myTreeView.SelectedNode.LastNode == null)
             {
-               string primitive_name = myTreeView.SelectedNode.Parent.Tag as string ?? myTreeView.SelectedNode.Parent.Text;
-               string sn = primitive_name + "()";
+               string sn = myTreeView.SelectedNode.Parent.Text + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
-      private void myHandleTreeViewKeyPress(object? sender, KeyPressEventArgs e)
-      {
-         if (!Char.IsLetterOrDigit(e.KeyChar))
-            return;
-
-         char search_char = Char.ToLowerInvariant(e.KeyChar);
-         TreeNode? assembly_node = _get_selected_assembly_node();
-         if (assembly_node is null)
-            return;
-
-         int start_idx = 0;
-         if (_last_primitive_search_char == search_char &&
-             ReferenceEquals(_last_primitive_search_assembly_node, assembly_node) &&
-             myTreeView.SelectedNode?.Parent == assembly_node)
-         {
-            start_idx = myTreeView.SelectedNode.Index + 1;
-         }
-
-         TreeNode? match_node = _find_matching_primitive_node(assembly_node, search_char, start_idx) ??
-                                _find_matching_primitive_node(assembly_node, search_char, 0);
-
-         if (match_node is not null)
-         {
-            myTreeView.SelectedNode = match_node;
-            match_node.EnsureVisible();
-            _last_primitive_search_char = search_char;
-            _last_primitive_search_assembly_node = assembly_node;
-            e.Handled = true;
-         }
-      }
-
-      private TreeNode? _get_selected_assembly_node()
-      {
-         if (myTreeView.SelectedNode is null)
-            return myTreeView.Nodes.Count > 0 ? myTreeView.Nodes[0] : null;
-
-         return myTreeView.SelectedNode.Parent ?? myTreeView.SelectedNode;
-      }
-
-      private TreeNode? _find_matching_primitive_node(TreeNode assembly_node, char search_char, int start_idx)
-      {
-         for (int i = start_idx; i < assembly_node.Nodes.Count; i++)
-         {
-            TreeNode primitive_node = assembly_node.Nodes[i];
-            string primitive_name = primitive_node.Tag as string ?? primitive_node.Text;
-            if (primitive_name.Length > 0 && Char.ToLowerInvariant(primitive_name[0]) == search_char)
-               return primitive_node;
-         }
-         return null;
-      }
-
-      private void myHandleTreeNodeMouseHover(object? sender, TreeNodeMouseHoverEventArgs e)
-      {
-         if (e.Node?.Tag is not string primitive_name)
-         {
-            _primitive_tool_tip.Hide(myTreeView);
-            return;
-         }
-
-         Type? primitive_type = PrimitiveManager.instance.get_dsl_op_type(primitive_name);
-         if (primitive_type is null)
-         {
-            _primitive_tool_tip.Hide(myTreeView);
-            return;
-         }
-
-         string tooltip_text = $"Namespace: {primitive_type.Namespace ?? "(none)"}";
-         string? source_path = _get_source_file_path(primitive_type);
-         if (!String.IsNullOrWhiteSpace(source_path))
-            tooltip_text += $"\nSource: {source_path}";
-
-         Point mouse_point = myTreeView.PointToClient(Cursor.Position);
-         _primitive_tool_tip.Show(tooltip_text, myTreeView, mouse_point.X + 12, mouse_point.Y + 12, 8000);
-      }
-
-      private void myHandleTreeViewMouseLeave(object? sender, EventArgs e)
-      {
-         _primitive_tool_tip.Hide(myTreeView);
-      }
-
-      private string? _get_source_file_path(Type primitive_type)
-      {
-         string cache_key = primitive_type.FullName ?? primitive_type.Name;
-         if (_primitive_source_path_cache.TryGetValue(cache_key, out string? cached_path))
-            return cached_path;
-
-         Dictionary<string, string?> source_file_index = _source_file_index ??= _build_source_file_index();
-         if (!source_file_index.TryGetValue(primitive_type.Name, out string? source_path))
-            return null;
-         _primitive_source_path_cache[cache_key] = source_path;
-         return source_path;
-      }
-
-      private Dictionary<string, string?> _build_source_file_index()
-      {
-         Dictionary<string, string?> source_file_index = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
-         string? repo_root = _repo_root_path ??= _find_repo_root();
-         if (String.IsNullOrWhiteSpace(repo_root))
-            return source_file_index;
-
-         string source_root = Path.Combine(repo_root, "src", "DataStreamSuite");
-         if (!Directory.Exists(source_root))
-            return source_file_index;
-
-         foreach (string file_path in Directory.EnumerateFiles(source_root, "*.cs", SearchOption.AllDirectories))
-         {
-            string type_name = Path.GetFileNameWithoutExtension(file_path);
-            if (!source_file_index.ContainsKey(type_name))
-               source_file_index[type_name] = file_path;
-         }
-         return source_file_index;
-      }
-
-      private string? _find_repo_root()
-      {
-         string? curr_path = AppDomain.CurrentDomain.BaseDirectory;
-         while (!String.IsNullOrWhiteSpace(curr_path))
-         {
-            if (Directory.Exists(Path.Combine(curr_path, "src", "DataStreamSuite")))
-               return curr_path;
-            curr_path = Directory.GetParent(curr_path)?.FullName;
-         }
-         return null;
-      }
-
       private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
       {
          Close();
@@ -1142,7 +681,7 @@ namespace BCG
          DialogResult dr = ofd.ShowDialog(this);
          if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
          {
-            PrimitiveManager.instance.load_streams_from_assembly(ofd.FileName);
+            PrimitiveManager.instance.loadAssembly(ofd.FileName);
             myLoadAssemblies();
          }
       }
@@ -1229,8 +768,7 @@ namespace BCG
          s = new MutableAppendStream(s, s);
          s.onChange += OnChange;
 
-         void OnChange(object? obj)
-         {
+         void OnChange(object? obj) {
             s.onChange -= OnChange;
             //s = new MutableAppendStream(s, s);
             var rowCollection = ((IDataPacket)obj!).newRows;

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -8,6 +8,7 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
+using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;
 
@@ -25,6 +26,10 @@ static class Program {
          host = HostBuilderExtensions.CreateBtsBuilder(args)
             .ConfigureServices(ConfigureServices)
             .Build();
+         var mdClient = host.Services.GetService<IMarketDataClient>();
+         if (mdClient is not null) {
+            PrimitiveManager.SetMarketDataClient(mdClient);
+         }
          await host.StartAsync();
          var winFormsThread = new Thread(RunWinFormsApp);
          winFormsThread.SetApartmentState(ApartmentState.STA);

src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json

@@ -5,7 +5,8 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
-  "Spark": {},
+  "Spark": {
+  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -52,16 +53,21 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
         },
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
+         "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }
   }
-}
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -13,8 +13,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "PubSub\PubSub.csp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSubServer", "PubSubServer\PubSubServer.csproj", "{0DBD814A-E17F-4836-8EEE-6E91B00C578F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersistenceServer", "PersistenceServer\PersistenceServer.csproj", "{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "BooleanExpressions\BooleanExpressions.csproj", "{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{88E70136-91AE-4F56-B243-8F4642B46D6A}"
@@ -45,10 +43,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSPublisher", "DSPublisher\DSPublisher.csproj", "{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIBridgeServer", "APIBridgeServer\APIBridgeServer.csproj", "{3B14C417-531A-4E39-80C1-A4C14BD000D5}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -119,18 +113,6 @@ Global
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x64.Build.0 = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.ActiveCfg = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.Build.0 = Release|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -311,30 +293,6 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -27,7 +27,6 @@
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
   </ItemGroup>
 
 

src/DataStreamSuite/DataStreamTests/HeikinAshiStreamTests.cs

@@ -1,121 +0,0 @@
-namespace Bts.DataStreamTests;
-
-using BCG;
-
-using BlueOrb.Operators.Signal;
-
-using Bts.DataStreamCore;
-
-[TestFixture]
-public class HeikinAshiStreamTests {
-   readonly BlueTableSchema schema = new(
-      new ColumnInfo[] {
-         new("id", 0, TypeCode.Int32),
-         new("ts_event", 1, TypeCode.DateTime),
-         new("open", 2, TypeCode.Double),
-         new("high", 3, TypeCode.Double),
-         new("low", 4, TypeCode.Double),
-         new("close", 5, TypeCode.Double),
-         new("volume", 6, TypeCode.Double),
-      }, "bars");
-
-   [Test]
-   public void AppendsHistoryAndCalculatesRecursiveHeikinAshiValues() {
-      var driver = new TestDriverStream(schema, null);
-      var heikinAshi = new HeikinAshiStream(driver);
-      var packets = new List<IDataPacket>();
-      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(2));
-
-      Assert.That(packets[0].newRows.Count, Is.EqualTo(1));
-      Assert.That(packets[0].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[0].deletedRows.Count, Is.EqualTo(0));
-
-      var first = packets[0].newRows[0];
-      Assert.That(first.get<double>("open"), Is.EqualTo(100.0).Within(1e-9));
-      Assert.That(first.get<double>("high"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(first.get<double>("low"), Is.EqualTo(90.0).Within(1e-9));
-      Assert.That(first.get<double>("close"), Is.EqualTo(105.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_open"), Is.EqualTo(102.5).Within(1e-9));
-      Assert.That(first.get<double>("ha_high"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_low"), Is.EqualTo(90.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_close"), Is.EqualTo(101.25).Within(1e-9));
-
-      Assert.That(packets[1].newRows.Count, Is.EqualTo(2));
-      Assert.That(packets[1].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[1].deletedRows.Count, Is.EqualTo(0));
-
-      var second = packets[1].newRows[1];
-      Assert.That(second.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
-      Assert.That(second.get<double>("high"), Is.EqualTo(112.0).Within(1e-9));
-      Assert.That(second.get<double>("low"), Is.EqualTo(104.0).Within(1e-9));
-      Assert.That(second.get<double>("close"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(second.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(second.get<double>("ha_high"), Is.EqualTo(112.0).Within(1e-9));
-      Assert.That(second.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(second.get<double>("ha_close"), Is.EqualTo(108.0).Within(1e-9));
-      Assert.That(second.get<double>("volume"), Is.EqualTo(20.0).Within(1e-9));
-   }
-
-   [Test]
-   public void RebuildsHistoryWhenSourceSnapshotChanges() {
-      var driver = new TestDriverStream(schema, null);
-      var heikinAshi = new HeikinAshiStream(driver);
-      var packets = new List<IDataPacket>();
-      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
-      );
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 114.0, 103.0, 111.0, 21.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(3));
-
-      Assert.That(packets[2].newRows.Count, Is.EqualTo(2));
-      Assert.That(packets[2].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[2].deletedRows.Count, Is.EqualTo(1));
-
-      var updated = packets[2].newRows[1];
-      Assert.That(updated.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
-      Assert.That(updated.get<double>("high"), Is.EqualTo(114.0).Within(1e-9));
-      Assert.That(updated.get<double>("low"), Is.EqualTo(103.0).Within(1e-9));
-      Assert.That(updated.get<double>("close"), Is.EqualTo(111.0).Within(1e-9));
-      Assert.That(updated.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(updated.get<double>("ha_high"), Is.EqualTo(114.0).Within(1e-9));
-      Assert.That(updated.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(updated.get<double>("ha_close"), Is.EqualTo(108.5).Within(1e-9));
-      Assert.That(updated.get<double>("volume"), Is.EqualTo(21.0).Within(1e-9));
-   }
-
-   static IBlueRow Row(int id, DateTime tsEvent, double open, double high, double low, double close, double volume) {
-      var row = new MutableBlueRow(
-         new BlueTableSchema(new[] {
-            new ColumnInfo("id", 0, TypeCode.Int32),
-            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
-            new ColumnInfo("open", 2, TypeCode.Double),
-            new ColumnInfo("high", 3, TypeCode.Double),
-            new ColumnInfo("low", 4, TypeCode.Double),
-            new ColumnInfo("close", 5, TypeCode.Double),
-            new ColumnInfo("volume", 6, TypeCode.Double),
-         }, "bars"));
-      row.setValue("id", id);
-      row.setValue("ts_event", tsEvent);
-      row.setValue("open", open);
-      row.setValue("high", high);
-      row.setValue("low", low);
-      row.setValue("close", close);
-      row.setValue("volume", volume);
-      return row;
-   }
-}

src/DataStreamSuite/DataStreamTests/TestDriverStream.cs

@@ -5,7 +5,7 @@ using BCG.Utilities.General;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
+using global::Spark.Core.Collections;
 
 public class TestDriverStream : ViewStream, IWorker {
    readonly IBlueTableSchema schema;

src/DataStreamSuite/DataStreamTests/VWAPStreamTests.cs

@@ -1,53 +0,0 @@
-namespace Bts.DataStreamTests;
-
-using BCG;
-
-using BlueOrb.Operators.Signal;
-
-using Bts.DataStreamCore;
-
-[TestFixture]
-public class VWAPStreamTests {
-   readonly BlueTableSchema schema = new(
-      new ColumnInfo[] {
-         new("id", 0, TypeCode.Int32),
-         new("ts_event", 1, TypeCode.DateTime),
-         new("close", 2, TypeCode.Double),
-         new("volume", 3, TypeCode.Double),
-      }, "bars");
-
-   [Test]
-   public void EmitsRowsAndCalculatesRollingVWAP() {
-      var driver = new TestDriverStream(schema, null);
-      var vwap = new VWAPStream(driver, "close", "volume", 2);
-      var packets = new List<IDataPacket>();
-      vwap.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 30, 5, DateTimeKind.Utc), 110.0, 20.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(1));
-      Assert.That(packets[0].newRows.Count, Is.EqualTo(2));
-
-      var rows = packets[0].newRows.to_list().OrderBy(r => r.get<int>("id")).ToList();
-      Assert.That(rows[0].get<double>("vwap"), Is.EqualTo(100.0).Within(1e-9));
-      Assert.That(rows[1].get<double>("vwap"), Is.EqualTo((100.0 * 10.0 + 110.0 * 20.0) / 30.0).Within(1e-9));
-   }
-
-   static IBlueRow Row(int id, DateTime tsEvent, double close, double volume) {
-      var row = new MutableBlueRow(
-         new BlueTableSchema(new[] {
-            new ColumnInfo("id", 0, TypeCode.Int32),
-            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
-            new ColumnInfo("close", 2, TypeCode.Double),
-            new ColumnInfo("volume", 3, TypeCode.Double),
-         }, "bars"));
-      row.setValue("id", id);
-      row.setValue("ts_event", tsEvent);
-      row.setValue("close", close);
-      row.setValue("volume", volume);
-      return row;
-   }
-}

src/DataStreamSuite/PersistenceServer/ConfigLoader.cs

@@ -1,33 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-
-public static class ConfigLoader {
-   public static PersistenceServerConfig load() {
-      var config = new PersistenceServerConfig();
-      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.json"));
-      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.Development.json"));
-      return config;
-   }
-
-   private static void merge_from_file(PersistenceServerConfig target, string path) {
-      if (!File.Exists(path)) {
-         return;
-      }
-
-      var json = File.ReadAllText(path);
-      var loaded = JsonSerializer.Deserialize<PersistenceServerConfig>(json, new JsonSerializerOptions {
-         PropertyNameCaseInsensitive = true
-      });
-
-      if (loaded?.pub_sub is null) {
-         return;
-      }
-
-      if (target.pub_sub is null) {
-         target.pub_sub = loaded.pub_sub;
-      } else {
-         target.pub_sub.MergeWith(loaded.pub_sub);
-      }
-   }
-}

src/DataStreamSuite/PersistenceServer/PeristenceHelper.cs

@@ -1,148 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Data.SQLite;
-using System.IO;
-using System.Linq;
-
-public class PeristenceHelper {
-   private readonly string _connectionString;
-   public string database_path { get; }
-
-   public PeristenceHelper(string? databasePath = null) {
-      var resolvedPath = databasePath ?? Path.Combine(AppContext.BaseDirectory, "PersistenceServer.sqlite");
-      database_path = resolvedPath;
-      Directory.CreateDirectory(Path.GetDirectoryName(resolvedPath)!);
-      _connectionString = $"Data Source={resolvedPath};Version=3";
-   }
-
-   public void InsertRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var columns = row.Keys.Select(QuoteIdentifier).ToArray();
-      var parameterNames = row.Keys.Select(CreateParameterName).ToArray();
-      var commandText =
-         $"INSERT INTO {QuoteIdentifier(tableName)} ({string.Join(", ", columns)}) VALUES ({string.Join(", ", parameterNames)})";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      AddParameters(command, row);
-      command.ExecuteNonQuery();
-   }
-
-   public void UpdateRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var keyColumns = GetKeyColumns(row);
-      var valueColumns = row.Keys.Where(column => !keyColumns.Contains(column, StringComparer.OrdinalIgnoreCase)).ToArray();
-      if (valueColumns.Length == 0) {
-         throw new InvalidOperationException("UpdateRow requires at least one non-key column to update.");
-      }
-
-      var setClause = string.Join(", ", valueColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateParameterName(column)}"));
-      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
-      var commandText = $"UPDATE {QuoteIdentifier(tableName)} SET {setClause} WHERE {whereClause}";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      foreach (var column in valueColumns) {
-         command.Parameters.AddWithValue(CreateParameterName(column), row[column]);
-      }
-
-      foreach (var column in keyColumns) {
-         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
-      }
-
-      command.ExecuteNonQuery();
-   }
-
-   public void RemoveRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var keyColumns = GetKeyColumns(row);
-      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
-      var commandText = $"DELETE FROM {QuoteIdentifier(tableName)} WHERE {whereClause}";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      foreach (var column in keyColumns) {
-         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
-      }
-
-      command.ExecuteNonQuery();
-   }
-
-   private SQLiteConnection OpenConnection() {
-      var connection = new SQLiteConnection(_connectionString);
-      connection.Open();
-      return connection;
-   }
-
-   private static void ValidateRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      if (string.IsNullOrWhiteSpace(tableName)) {
-         throw new ArgumentException("Table name is required.", nameof(tableName));
-      }
-
-      if (row.Count == 0) {
-         throw new ArgumentException("Row must contain at least one column.", nameof(row));
-      }
-   }
-
-   private static string[] GetKeyColumns(IReadOnlyDictionary<string, string> row) {
-      if (row.ContainsKey("namespace") && row.ContainsKey("key")) {
-         return new[] { "namespace", "key" };
-      }
-
-      if (row.ContainsKey("key")) {
-         return new[] { "key" };
-      }
-
-      throw new InvalidOperationException("Row must contain 'key', or both 'namespace' and 'key', for update/delete.");
-   }
-
-   private static void EnsureTable(SQLiteConnection connection, string tableName, IEnumerable<string> columns) {
-      var existingColumns = GetExistingColumns(connection, tableName);
-      if (existingColumns.Count == 0) {
-         var createColumns = string.Join(", ", columns.Select(column => $"{QuoteIdentifier(column)} TEXT"));
-         using var createCommand =
-            new SQLiteCommand($"CREATE TABLE IF NOT EXISTS {QuoteIdentifier(tableName)} ({createColumns})", connection);
-         createCommand.ExecuteNonQuery();
-         existingColumns = GetExistingColumns(connection, tableName);
-      }
-
-      foreach (var column in columns) {
-         if (existingColumns.Contains(column)) {
-            continue;
-         }
-
-         using var alterCommand =
-            new SQLiteCommand($"ALTER TABLE {QuoteIdentifier(tableName)} ADD COLUMN {QuoteIdentifier(column)} TEXT", connection);
-         alterCommand.ExecuteNonQuery();
-      }
-   }
-
-   private static HashSet<string> GetExistingColumns(SQLiteConnection connection, string tableName) {
-      var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-      using var command = new SQLiteCommand($"PRAGMA table_info({QuoteIdentifier(tableName)})", connection);
-      using var reader = command.ExecuteReader();
-      while (reader.Read()) {
-         columns.Add(reader["name"].ToString()!);
-      }
-
-      return columns;
-   }
-
-   private static void AddParameters(SQLiteCommand command, IReadOnlyDictionary<string, string> row) {
-      foreach (var pair in row) {
-         command.Parameters.AddWithValue(CreateParameterName(pair.Key), pair.Value);
-      }
-   }
-
-   private static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
-
-   private static string CreateParameterName(string columnName) => $"@p_{columnName.Replace("\"", string.Empty)}";
-
-   private static string CreateWhereParameterName(string columnName) => $"@w_{columnName.Replace("\"", string.Empty)}";
-}

src/DataStreamSuite/PersistenceServer/PersistenceCallHandler.cs

@@ -1,155 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Net;
-using System.Text.Json;
-
-public class PersistenceCallHandler {
-   private const string KeyValueTableName = "key_value_store";
-   private readonly PeristenceHelper _persistenceHelper;
-   private readonly PubSubSync _pub_sub_sync;
-
-   public PersistenceCallHandler(PeristenceHelper persistenceHelper, PubSubSync pub_sub_sync) {
-      _persistenceHelper = persistenceHelper;
-      _pub_sub_sync = pub_sub_sync;
-   }
-
-   public void Handle(HttpListenerRequest request, string payload) {
-      Console.WriteLine("----- Incoming REST call -----");
-      Console.WriteLine($"Timestamp: {DateTimeOffset.Now:O}");
-      Console.WriteLine($"Method: {request.HttpMethod}");
-      Console.WriteLine($"Url: {request.Url}");
-      Console.WriteLine($"RawUrl: {request.RawUrl}");
-      Console.WriteLine($"RemoteEndPoint: {request.RemoteEndPoint}");
-      Console.WriteLine($"UserAgent: {request.UserAgent}");
-      Console.WriteLine($"ContentType: {request.ContentType}");
-      Console.WriteLine($"ContentLength: {request.ContentLength64}");
-
-      if (request.QueryString.Count > 0) {
-         Console.WriteLine("Query:");
-         foreach (var key in request.QueryString.AllKeys) {
-            if (key is null) {
-               continue;
-            }
-
-            Console.WriteLine($"  {key} = {request.QueryString[key]}");
-         }
-      }
-
-      if (request.Headers.Count > 0) {
-         Console.WriteLine("Headers:");
-         foreach (var headerKey in request.Headers.AllKeys) {
-            if (headerKey is null) {
-               continue;
-            }
-
-            Console.WriteLine($"  {headerKey}: {request.Headers[headerKey]}");
-         }
-      }
-
-      Console.WriteLine("Body:");
-      Console.WriteLine(string.IsNullOrWhiteSpace(payload) ? "  <empty>" : payload);
-
-      var rpcRequest = ParseRpcRequest(payload);
-      HandleRpc(rpcRequest.FunctionName, rpcRequest.Arguments);
-      Console.WriteLine("------------------------------");
-   }
-
-   private static PersistenceRpcRequest ParseRpcRequest(string payload) {
-      if (string.IsNullOrWhiteSpace(payload)) {
-         throw new InvalidOperationException("RPC payload is empty.");
-      }
-
-      using var document = JsonDocument.Parse(payload);
-      var root = document.RootElement;
-
-      if (!root.TryGetProperty("function_name", out var functionNameElement) ||
-          functionNameElement.ValueKind != JsonValueKind.String) {
-         throw new InvalidOperationException("RPC payload must contain a string 'function_name'.");
-      }
-
-      if (!root.TryGetProperty("arguments", out var argumentsElement) ||
-          argumentsElement.ValueKind != JsonValueKind.Array) {
-         throw new InvalidOperationException("RPC payload must contain an 'arguments' array.");
-      }
-
-      var arguments = new List<string>();
-      foreach (var argument in argumentsElement.EnumerateArray()) {
-         if (argument.ValueKind != JsonValueKind.String) {
-            throw new InvalidOperationException("RPC arguments must all be strings.");
-         }
-
-         arguments.Add(argument.GetString()!);
-      }
-
-      return new PersistenceRpcRequest(functionNameElement.GetString()!, arguments.ToArray());
-   }
-
-   private void HandleRpc(string functionName, params string[] arguments) {
-      switch (functionName) {
-         case "keyvalue_insert":
-            RequireArgumentCount(functionName, arguments, 3);
-            HandleKeyValueInsert(arguments[0], arguments[1], arguments[2]);
-            break;
-         case "keyvalue_update":
-            RequireArgumentCount(functionName, arguments, 3);
-            HandleKeyValueUpdate(arguments[0], arguments[1], arguments[2]);
-            break;
-         case "keyvalue_delete":
-            RequireArgumentCount(functionName, arguments, 2);
-            HandleKeyValueDelete(arguments[0], arguments[1]);
-            break;
-         default:
-            throw new InvalidOperationException($"Unknown RPC function '{functionName}'.");
-      }
-   }
-
-   private static void RequireArgumentCount(string functionName, string[] arguments, int expectedCount) {
-      if (arguments.Length != expectedCount) {
-         throw new InvalidOperationException(
-            $"RPC function '{functionName}' expected {expectedCount} arguments but received {arguments.Length}.");
-      }
-   }
-
-   private void HandleKeyValueInsert(string itemNamespace, string key, string value) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-         ["value"] = value,
-      };
-      _persistenceHelper.InsertRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_insert(namespace='{itemNamespace}', key='{key}', value='{value}')");
-   }
-
-   private void HandleKeyValueUpdate(string itemNamespace, string key, string value) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-         ["value"] = value,
-      };
-      _persistenceHelper.UpdateRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_update(namespace='{itemNamespace}', key='{key}', value='{value}')");
-   }
-
-   private void HandleKeyValueDelete(string itemNamespace, string key) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-      };
-      _persistenceHelper.RemoveRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_delete(namespace='{itemNamespace}', key='{key}')");
-   }
-}
-
-public record PersistenceRpcRequest {
-   public PersistenceRpcRequest(string functionName, string[] arguments) {
-      FunctionName = functionName;
-      Arguments = arguments;
-      Console.WriteLine($"PersistenceRpcRequest created for function '{FunctionName}' with {Arguments.Length} argument(s).");
-   }
-
-   public string FunctionName { get; init; }
-   public string[] Arguments { get; init; }
-}

src/DataStreamSuite/PersistenceServer/PersistenceServer.csproj

@@ -1,23 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Content Include="appsettings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/PersistenceServer/PersistenceServerConfig.cs

@@ -1,5 +0,0 @@
-using Bts.PubSub;
-
-public class PersistenceServerConfig {
-   public PubSubConfig? pub_sub { get; set; }
-}

src/DataStreamSuite/PersistenceServer/Program.cs

@@ -1,113 +0,0 @@
-using Bts.DataStreamCore;
-using BCG;
-
-using System;
-using System.IO;
-using System.Net;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-
-const int Port = 5079;
-var prefix = $"http://localhost:{Port}/";
-
-var listener = new HttpListener();
-listener.Prefixes.Add(prefix);
-var config = ConfigLoader.load();
-var persistenceHelper = new PeristenceHelper();
-var pub_sub_sync = new PubSubSync(persistenceHelper.database_path, config.pub_sub);
-var callHandler = new PersistenceCallHandler(persistenceHelper, pub_sub_sync);
-
-Console.WriteLine("PersistenceServer starting.");
-Console.WriteLine($"Listening on {prefix}");
-Console.WriteLine($"DataStreamCore loaded from: {typeof(IDataPacket).Assembly.FullName}");
-Console.WriteLine("Press Ctrl+C to stop.");
-
-var shutdown = new CancellationTokenSource();
-
-Console.CancelKeyPress += (_, eventArgs) => {
-   eventArgs.Cancel = true;
-   Console.WriteLine("Shutdown requested.");
-   shutdown.Cancel();
-   if (listener.IsListening) {
-      listener.Stop();
-   }
-};
-
-try {
-   pub_sub_sync.publish_all_tables();
-   listener.Start();
-   await RunAsync(listener, callHandler, shutdown.Token);
-   return 0;
-} catch (HttpListenerException ex) when (shutdown.IsCancellationRequested) {
-   Console.WriteLine($"Listener stopped: {ex.Message}");
-   return 0;
-} catch (Exception ex) {
-   Console.Error.WriteLine($"PersistenceServer crashed: {ex}");
-   return 1;
-} finally {
-   listener.Close();
-}
-
-static async Task RunAsync(
-   HttpListener listener,
-   PersistenceCallHandler callHandler,
-   CancellationToken cancellationToken) {
-   while (!cancellationToken.IsCancellationRequested) {
-      HttpListenerContext context;
-      try {
-         context = await listener.GetContextAsync();
-      } catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) {
-         break;
-      } catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) {
-         break;
-      }
-
-      _ = Task.Run(() => HandleRequestAsync(context, callHandler, cancellationToken), cancellationToken);
-   }
-}
-
-static async Task HandleRequestAsync(
-   HttpListenerContext context,
-   PersistenceCallHandler callHandler,
-   CancellationToken cancellationToken) {
-   var request = context.Request;
-   var response = context.Response;
-
-   try {
-      var body = await ReadBodyAsync(request, cancellationToken);
-      callHandler.Handle(request, body);
-
-      var responseText =
-         $$"""
-         {"status":"ok","method":"{{request.HttpMethod}}","path":"{{request.RawUrl ?? "/"}}"}
-         """;
-
-      var bytes = Encoding.UTF8.GetBytes(responseText);
-      response.StatusCode = (int)HttpStatusCode.OK;
-      response.ContentType = "application/json";
-      response.ContentEncoding = Encoding.UTF8;
-      response.ContentLength64 = bytes.LongLength;
-      await response.OutputStream.WriteAsync(bytes, cancellationToken);
-   } catch (Exception ex) {
-      Console.Error.WriteLine($"Request handling failed: {ex}");
-      response.StatusCode = (int)HttpStatusCode.InternalServerError;
-   } finally {
-      response.OutputStream.Close();
-      response.Close();
-   }
-}
-
-static async Task<string> ReadBodyAsync(HttpListenerRequest request, CancellationToken cancellationToken) {
-   if (!request.HasEntityBody) {
-      return string.Empty;
-   }
-
-   using var reader = new StreamReader(
-      request.InputStream,
-      request.ContentEncoding ?? Encoding.UTF8,
-      detectEncodingFromByteOrderMarks: true,
-      leaveOpen: true);
-
-   return await reader.ReadToEndAsync(cancellationToken);
-}

src/DataStreamSuite/PersistenceServer/PubSubSync.cs

@@ -1,116 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Data;
-using System.Data.SQLite;
-
-using BCG;
-using Bts.DataStreamCore;
-using Bts.PubSub;
-using Bts.PubSub.Routing;
-using Microsoft.Extensions.DependencyInjection;
-
-using bts;
-
-public class PubSubSync {
-   private readonly string _connectionString;
-   private readonly Dictionary<string, IPubTableBroker> _brokers = new(StringComparer.OrdinalIgnoreCase);
-   private readonly object _sync_root = new();
-   private readonly IServiceProvider _serviceProvider;
-   private readonly PubSubRouterConfig _router_config;
-   private readonly PubSubTagHandlerFactoryConfig _tag_handler_factory_config;
-
-   public PubSubSync(string databasePath, PubSubConfig? pubSubConfig = null) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(databasePath);
-      _connectionString = $"Data Source={databasePath};Version=3";
-      var resolved_pub_sub_config = get_or_initialize_pub_sub_config(pubSubConfig);
-      _router_config = resolved_pub_sub_config.Routing ?? throw new InvalidOperationException("PubSub routing config is required.");
-      _tag_handler_factory_config = resolved_pub_sub_config.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default();
-      _serviceProvider = new ServiceCollection().BuildServiceProvider();
-   }
-
-   public void publish_all_tables() {
-      lock (_sync_root) {
-         foreach (var table_name in get_table_names()) {
-            publish_table_core(table_name);
-         }
-      }
-   }
-
-   public void publish_table(string table_name) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(table_name);
-
-      lock (_sync_root) {
-         publish_table_core(table_name);
-      }
-   }
-
-   private void publish_table_core(string table_name) {
-      var rows = load_table(table_name);
-      var broker = get_or_create_broker(table_name);
-      broker.create_table(rows);
-      Console.WriteLine($"PubSubSync published table '{table_name}' with {rows.Count} row(s).");
-   }
-
-   private IPubTableBroker get_or_create_broker(string table_name) {
-      if (_brokers.TryGetValue(table_name, out var broker)) {
-         return broker;
-      }
-
-      var publisher_core = new PublisherCore(
-         new PubSubRouter(_router_config),
-         new PubsubTagHandlerFactory(_serviceProvider, _tag_handler_factory_config));
-      broker = new PubTableBroker(publisher_core);
-      _brokers[table_name] = broker;
-      return broker;
-   }
-
-   private string[] get_table_names() {
-      using var connection = open_connection();
-      using var command = new SQLiteCommand(
-         "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-         connection);
-      using var reader = command.ExecuteReader();
-
-      var table_names = new List<string>();
-      while (reader.Read()) {
-         table_names.Add(reader.GetString(0));
-      }
-
-      return table_names.ToArray();
-   }
-
-   private IBlueRowCollection load_table(string table_name) {
-      using var connection = open_connection();
-      using var adapter = new SQLiteDataAdapter($"SELECT * FROM {quote_identifier(table_name)}", connection);
-      var table = new DataTable(table_name);
-      adapter.Fill(table);
-      table.TableName = table_name;
-      return BlueDataUtility.toIBlueRowCollection(table);
-   }
-
-   private SQLiteConnection open_connection() {
-      var connection = new SQLiteConnection(_connectionString);
-      connection.Open();
-      return connection;
-   }
-
-   private static PubSubConfig get_or_initialize_pub_sub_config(PubSubConfig? configured_pub_sub) {
-      try {
-         PubSubConfig.SetGlobalConfig(PubSubConfig.Default());
-         var existing = PubSubConfig.Instance;
-         if (configured_pub_sub is not null) {
-            existing.MergeWith(configured_pub_sub);
-         }
-         return existing;
-      } catch (InvalidOperationException) {
-         var config = PubSubConfig.Default();
-         if (configured_pub_sub is not null) {
-            config.MergeWith(configured_pub_sub);
-         }
-         PubSubConfig.SetGlobalConfig(config);
-         return config;
-      }
-   }
-
-   private static string quote_identifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
-}

src/DataStreamSuite/PersistenceServer/appsettings.json

@@ -1,20 +0,0 @@
-{
-  "PubSub": {
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    }
-  }
-}

src/DataStreamSuite/PersistenceServer/client/persistence_client.py

@@ -1,43 +0,0 @@
-import json
-import sys
-import urllib.error
-import urllib.request
-
-
-SERVER_URL = "http://localhost:5079/save?source=python-client"
-
-
-def main() -> int:
-   payload = {
-      "function_name": "keyvalue_insert",
-      "arguments": [
-         "example-namespace",
-         "example-key",
-         "hello from python",
-      ],
-   }
-
-   data = json.dumps(payload).encode("utf-8")
-   request = urllib.request.Request(
-      SERVER_URL,
-      data=data,
-      headers={
-         "Content-Type": "application/json",
-         "X-Client": "PersistenceServerPythonClient",
-      },
-      method="POST",
-   )
-
-   try:
-      with urllib.request.urlopen(request) as response:
-         body = response.read().decode("utf-8")
-         print(f"status: {response.status}")
-         print(f"body: {body}")
-         return 0
-   except urllib.error.URLError as error:
-      print(f"request failed: {error}", file=sys.stderr)
-      return 1
-
-
-if __name__ == "__main__":
-   raise SystemExit(main())

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/ClientToServerConnectionWrapper.cs

@@ -1,44 +1,44 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.Runtime.InteropServices;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Runtime.InteropServices;
 
-// namespace BCG
-// {
-//    public class ClientToServerConnectionWrapper
-//    {
-//       [DllImport("BlutoInterop.dll")]
-//       public unsafe static extern void init();
+namespace BCG
+{
+   public class ClientToServerConnectionWrapper
+   {
+      [DllImport("BlutoInterop.dll")]
+      public unsafe static extern void init();
 
-//       [DllImport("BlutoInterop.dll")]
-//       public unsafe static extern void initServerConnection(byte[] ip, int port);
+      [DllImport("BlutoInterop.dll")]
+      public unsafe static extern void initServerConnection(byte[] ip, int port);
 
-//       [DllImport("BlutoInterop.dll")]
-//       public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
+      [DllImport("BlutoInterop.dll")]
+      public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
 
-//       [DllImport("BlutoInterop.dll")]
-//       public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
+      [DllImport("BlutoInterop.dll")]
+      public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
 
-//       public static byte[] stringToBytes(string s)
-//       {
-//          byte[] rv = new byte[s.Length + 1];
-//          for (int i = 0; i < s.Length; i++)
-//             rv[i] = Convert.ToByte(s[i]);
-//          rv[s.Length] = Convert.ToByte ('\0');
-//          return rv;
-//       }
+      public static byte[] stringToBytes(string s)
+      {
+         byte[] rv = new byte[s.Length + 1];
+         for (int i = 0; i < s.Length; i++)
+            rv[i] = Convert.ToByte(s[i]);
+         rv[s.Length] = Convert.ToByte ('\0');
+         return rv;
+      }
 
-//       public static string intPtrToString(IntPtr iptr)
-//       {
-//          return Marshal.PtrToStringAnsi(iptr);
-//       }
+      public static string intPtrToString(IntPtr iptr)
+      {
+         return Marshal.PtrToStringAnsi(iptr);
+      }
 
-//       public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
-//       {
-//          byte[] data = new byte[sz];
-//          Marshal.Copy(iptr, data, 0, data.Length);
-//          return data;
-//       }
+      public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
+      {
+         byte[] data = new byte[sz];
+         Marshal.Copy(iptr, data, 0, data.Length);
+         return data;
+      }
 
-//    }
-// }
+   }
+}

src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs

@@ -39,6 +39,8 @@ public static class PubSubMessageUtils {
    }
 }
 
+// Subscribe //////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
    public string? connection;
    public string subscribing_app = ClientInfo.thisClient.app;
@@ -84,6 +86,8 @@ public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
       $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {string.Concat(tags.empty_if_null())}";
 }
 
+// Unsubscribe ////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
    public bool Equals(UnsubscribeMsg? other) =>
       other is not null && base.Equals(other);
@@ -95,6 +99,8 @@ public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
       base.GetHashCode();
 }
 
+// Trasnaction /////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
    uint n_msgs;
 
@@ -121,12 +127,15 @@ public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
       n_msgs.GetHashCode();
 }
 
+// Publish ////////////////////////////////////////////////////////////////
+/// ///////////////////////////////////////////////////////////////////////
 public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
    public string? connection;
    public string publishing_app = ClientInfo.thisClient.app;
    public string? tag;
    public string? tag_type;
    public bool is_compressed = false;
+   // this is the size that of the msg, regarless of it being serialized or not
    public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
    public byte[]? data_to_publish;
 
@@ -135,6 +144,8 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
       Log.assert(!string.IsNullOrEmpty(tag_type));
       Log.assert(!string.IsNullOrEmpty(tag));
       Log.assert(!string.IsNullOrEmpty(connection));
+      // The 'ThrowIfNullOrEmpty' calls are redundant (given the assert statements above), but
+      // necessary for the compiler to recognize the strings won't be null when this method exits.
       ArgumentException.ThrowIfNullOrEmpty(tag_type);
       ArgumentException.ThrowIfNullOrEmpty(tag);
       ArgumentException.ThrowIfNullOrEmpty(connection);
@@ -170,7 +181,7 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
    }
 
    public bool Equals(PublishMsg? other) =>
-      other is not null
+      other is not null 
          && connection == other.connection
          && publishing_app == other.publishing_app
          && tag == other.tag
@@ -209,6 +220,7 @@ public enum PubSubMessageType {
 
 public class PubSubClientMessage : IPubSubMessage {
    public PubSubMessageType msg_type;
+   // only one of these wil be non null
    public SubscribeMsg? sub_msg;
    public UnsubscribeMsg? unsub_msg;
    public TransactionMsg? transaction_msg;
@@ -253,7 +265,7 @@ public class PubSubClientMessage : IPubSubMessage {
             ValidateMsg(pub_msg).serialize_into_str(str);
             break;
 
-         T ValidateMsg<T>([NotNull] T? msg, [CallerArgumentExpression(nameof(msg))] string name = "") {
+         T ValidateMsg<T>([NotNull]T? msg, [CallerArgumentExpression(nameof(msg))]string name = "") {
             if (msg is null) {
                throw new InvalidOperationException($"msg type is {msg_type} but {name} is null");
             }
@@ -371,7 +383,7 @@ public class PubSubServerMessage : IPubSubMessage, IEquatable<PubSubServerMessag
 
    [MemberNotNull(nameof(tag), nameof(tag_type))]
    public void construct_deserialize(byte[] msg) =>
-      construct_deserialize(new MemoryStream(msg) { Position = 0 });
+      construct_deserialize(new MemoryStream(msg) { Position = 0});
 
    public bool Equals(PubSubServerMessage? other) =>
       other is not null

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -48,16 +48,16 @@ public class PubSubConfig {
          HeartbeatInterval = 15000,
          MissingHeartbeatsBeforeGc = 50,
          DisposedConnectionCheckPeriod = 1000,
-         ServerPort = 22019,
+         ServerPort = 12345,
          TagTypeToAssemblyMap = new(),
          TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(),
          SimpleEchoServer = false,
          Routing = new() {
             PublisherRoutes = new() {
-               new("bts-cloud://127.0.0.1:22019", ".*")
+               new("bts-cloud://127.0.0.1:12345", ".*")
             },
             SubscriberRoutes = new() {
-               new("bts-cloud://127.0.0.1:22019", ".*")
+               new("bts-cloud://127.0.0.1:12345", ".*")
             }
          }
       };

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs

@@ -12,7 +12,6 @@ using System.Net.Sockets;
 
 // opens a TCP port and starts listening to connections
 public class AsyncTCPConnectionAcceptor_APM {
-   // This acceptor exists as an alternate implementation and is not the one currently used by the pubsub server.
    private readonly int _port;
    Socket? _accept_socket;
    readonly ITCPConnectionHandler _connection_handler;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs

@@ -13,7 +13,6 @@ namespace bts {
 
    // opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
    // it uses TcpListener, instead of raw sockets.
-   // This is the acceptor currently used by the pubsub server via ManagedTCPMessageHub.
    public class AsyncTCPConnectionAcceptor_AsyncWrapper {
       private readonly int _accept_port;
       // private Socket _accept_socket;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs

@@ -10,7 +10,6 @@ namespace bts {
     // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
     // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
     // statefullness and hence non static
-    // This connector exists as an alternate implementation and is not the one currently used by pubsub clients.
     public static class AsyncTCPConnector_APM {
 
         internal class SocketConnectionArgs {

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs

@@ -13,18 +13,17 @@ using System.Diagnostics.CodeAnalysis;
 
 
 namespace bts {
-   // Unlike AsyncTCPConnectionAcceptor_AsyncWrappe this class is static since the requests to connect are stand alone requests
-   // whereas AsyncTCPConnectionAcceptor_AsyncWrappe is always listening for incoming connections which lends itself to more
+   // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
+   // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
    // statefullness and hence non static
-   // This is the connector currently used by pubsub clients via PubSubServerManagedTCPConnection.
    public static class AsyncTCPConnector_AsyncWrapper {
       public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
          bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}");
          //Task.Run(() => _connect_to(server_address, handler));
          ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(
-            _connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
+            Connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
 
-         bool _connect() {
+         bool Connect() {
             try {
                return _connect_to(server_address, handler);
             } catch (Exception ex) {
@@ -50,7 +49,7 @@ namespace bts {
          handler.onConnection(conn);
          return true;
 
-         bool CreateTcpClient([NotNullWhen(true)] out TcpClient? tcpClient) {
+         bool CreateTcpClient([NotNullWhen(true)]out TcpClient? tcpClient) {
             try {
                tcpClient = new(server_rci.ip, server_rci.port);
                return true;

src/DataStreamSuite/PubSub/Misc/ClientInfo.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Text;
 
 namespace BCG
@@ -24,8 +24,8 @@ namespace BCG
       {
          get { return myApp; }
          set { myApp = value; }
-      }
 
+      }
       private string myKeyQualifier;
       public string keyQualifier
       {
@@ -38,8 +38,8 @@ namespace BCG
       {
          get { return myStartTime; }
          set { myStartTime = value; }
-      }
 
+      }
       string? myProcessString;
       public string processString
       {
@@ -49,8 +49,10 @@ namespace BCG
                myProcessString = getProcessString(";");
             return myProcessString;
          }
+
       }
 
+      
       public ClientInfo(string theMachineName, int thePid, string theProcessName, string theKeyQualifier, DateTime theStartTime)
       {
          myMachineName = theMachineName;
@@ -66,8 +68,10 @@ namespace BCG
          myMachineName = Environment.MachineName;
          myPid = System.Diagnostics.Process.GetCurrentProcess().Id;
          myApp = System.Diagnostics.Process.GetCurrentProcess().ProcessName.ToUpper();
-         string[] args = Environment.GetCommandLineArgs();
+         string[] args = System.Environment.GetCommandLineArgs();
 
+         //the key qualifier is the last argument given. If they didn't give any arguments
+         //but the program itself, then the KQ is empty. 
          if (args.Length > 1)
             myKeyQualifier = args[args.Length - 1];
          else
@@ -82,13 +86,13 @@ namespace BCG
 
       public static ClientInfo thisClient
       {
-         get
+         get 
          {
             if (myLocalClient == null)
             {
-               lock (myLock)
+               lock(myLock)
                {
-                  if (myLocalClient == null)
+                  if(myLocalClient == null)
                      myLocalClient = new ClientInfo();
                }
             }
@@ -98,6 +102,7 @@ namespace BCG
 
       public static ClientInfo fromProcessString(string procString)
       {
+
          ClientInfo outputGuy = new ClientInfo();
 
          string[] info = procString.Split(';');
@@ -107,7 +112,9 @@ namespace BCG
          outputGuy.myKeyQualifier = info[2];
          outputGuy.myPid = Int32.Parse(info[3]);
 
+
          return outputGuy;
+
       }
 
       private string getProcessString(string seperator)

src/DataStreamSuite/PubSub/Misc/IWorker.cs

@@ -3,7 +3,7 @@ using System;
 namespace BCG.Utilities.General
 {
    public interface IWorker
-   {
+	{
       void doWork();
-   }
+	}
 }

src/DataStreamSuite/PubSub/Misc/Pair.cs

@@ -79,4 +79,5 @@ namespace mbts
          return ToString().GetHashCode();
       }
    }
+
 }

src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs

@@ -9,7 +9,7 @@ namespace BCG {
    /// <summary>
    /// Collection of utility methods used throughout
    /// </summary>
-   public static class StreamUtilityMethods {
+	public static class StreamUtilityMethods {
 
       // extension method to write a string C++ style (len prefixed)
       public static void write_string_cpp(this BinaryWriter bw, string? str) {
@@ -98,7 +98,7 @@ namespace BCG {
          byte[] secondStreamBytes = new byte[stream.Length - splitLocation];
          for (int i = 0; i < stream.Length; ++i) {
             //for some resaon, the 'getBytesFromStream' function prepends the length of the byte array to the bytes. This means
-            //we have to ignore those bytes here.
+            //we have to ignore those bytes here. 
             if (i < splitLocation)
                firstStreamBytes[i] = streamBytes[i + 4];
             else
@@ -107,6 +107,8 @@ namespace BCG {
          }
 
          return new Stream[] { new MemoryStream(firstStreamBytes), new MemoryStream(secondStreamBytes) };
+
+
       }
    }
 }

src/DataStreamSuite/PubSub/PublishedCache.cs

@@ -4,8 +4,7 @@ using System;
 using System.Collections.Generic;
 
 namespace bts {
-   // This caches all the published data, on the pubsliher (client) side. 
-   // This class is created and held by
+   // This caches all the published data. This class is created and held by
    // PublisherCore. When data is publsihed , PublisherCore passes it to this 
    // class via the publish call. On a reconnection to the PubSubServer, 
    // PublisherCore calls get_content to get the data to re-publish, to 

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -43,20 +43,20 @@ namespace BCG {
 
       #region constructors
 
-      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tag_handler_factory) {
+      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tagHandlerFactory) {
          _router = router;
-         _state = new(tag_handler_factory);
+         _state = new(tagHandlerFactory);
          Thread pump_thread = new Thread(new ThreadStart(_message_pump));
          pump_thread.IsBackground = true;
          pump_thread.Start();
       }
 
-      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tag_handler_factory = null, IServiceProvider? service_provider = null) {
+      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null, IServiceProvider? serviceProvider = null) {
          router ??= new PubSubRouter();
-         tag_handler_factory ??= new PubsubTagHandlerFactory(
-            service_provider ?? PubSubServices.ServiceProvider,
+         tagHandlerFactory ??= new PubsubTagHandlerFactory(
+            serviceProvider ?? PubSubServices.ServiceProvider,
             PubSubConfig.Instance.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default());
-         return new(router, tag_handler_factory);
+         return new(router, tagHandlerFactory);
       }
 
       public void Dispose() { }
@@ -119,7 +119,7 @@ namespace BCG {
       #endregion
 
       // static void setGlobalInstance() {
-
+         
       // }
 
       #region Message Pump Thread - this is where messages are dequeued and the action performed

src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs

@@ -2,6 +2,6 @@ namespace Bts.PubSub.Server;
 
 using bts;
 
-public interface IInternalPublisher {
+interface IInternalPublisher {
    void Publish(PublishMsg message);
 }

src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs

@@ -10,7 +10,7 @@ using System.Collections.Generic;
 /// messages to be published. It is also passed to the <see cref="BCG.PubSubMessageReactor"/>, so it
 /// can dequeue these messages and send them over the wire.
 /// </summary>
-public interface IInternalPublisherQueue {
+interface IInternalPublisherQueue {
    void Enqueue(PublishMsg message);
    List<PublishMsg> DequeueMultiple(TimeSpan ts, int maxElements);
 }