Cleaning market data streams

Replay streams + TC2 position
launch
2026-06-09 22:58:04 -04:00 · 2026-04-21 14:46:20 -04:00 · 2026-04-13 23:46:47 -04:00 · 2026-04-13 23:46:21 -04:00 · 2026-02-21 16:36:53 -05:00 · 2026-02-21 16:36:53 -05:00
300 changed files with 20115 additions and 2879 deletions
@@ -1,5 +1,11 @@
 {
-  "dockerFile": "Dockerfile",
+  "build": {
+    "dockerfile": "../docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile",
+    "options": [
+      "--secret",
+      "id=bitbucket_access_token,src=${localWorkspaceFolder}/.secrets/bitbucket_access_token.txt"
+    ]
+  },
  "hostRequirements": {
    "memory": "8gb",
    "storage": "10gb"
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
@@ -0,0 +1,4 @@
+// <auto-generated> This file has been auto generated. </auto-generated>
+using System;
+[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
+class AutoGeneratedProgram {static void Main(string[] args){}}
@@ -64,3 +64,9 @@ trace_data/
 *.lst
 .env.local
 .secrets
+.dotnet-cli/
+.dotnet-cli/
+.dotnet-cli /
+.dotnet/
+.codex_restore*.log
+src/cs/Bts.Spark.Core/.codexobj/
@@ -1,25 +1,6 @@
 {
  "version": "0.2.0",
  "configurations": [
-  {
-    "name": "(gdb) Attach",
-    "type": "cppdbg",
-    "request": "attach",
-    "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-    "MIMode": "gdb",
-    "setupCommands": [
-        {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-        },
-        {
-            "description": "Set Disassembly Flavor to Intel",
-            "text": "-gdb-set disassembly-flavor intel",
-            "ignoreFailures": true
-        }
-    ]
-  },
    {
      "name": "Python Debugger: Current File with Arguments",
      "type": "debugpy",
@@ -161,17 +142,30 @@
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStream (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
        "cwd": "build/Debug/bin/net8.0-windows",
        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "DataStreamQueryAnalyzer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "cwd": "build/Debug/bin/net8.0-windows",
+        "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe",
+          "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
        },
      },
      "stopAtEntry": false,
@@ -209,9 +203,71 @@
        "program": "PubSubServer.exe",
      },
      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      // "env": {
+      //   "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      // },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DSPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DSPublisher",
      },
+      "windows": {
+        "program": "DSPublisher.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DSPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DSPublisher",
+      },
+      "windows": {
+        "program": "DSPublisher.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PersistenceServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PersistenceServer",
+      },
+      "windows": {
+        "program": "PersistenceServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PersistenceServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PersistenceServer",
+      },
+      "windows": {
+        "program": "PersistenceServer.exe",
+      },
+      "cwd": "build/Debug/bin/net8.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -390,6 +446,25 @@
        }
      ]
    },
+    {
+      "name": "Sparkd Attach",
+      "type": "cppdbg",
+      "request": "attach",
+      "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+      "MIMode": "gdb",
+      "setupCommands": [
+        {
+          "description": "Enable pretty-printing for gdb",
+          "text": "-enable-pretty-printing",
+          "ignoreFailures": true
+        },
+        {
+          "description": "Set Disassembly Flavor to Intel",
+          "text": "-gdb-set disassembly-flavor intel",
+          "ignoreFailures": true
+        }
+      ]
+    },
  ],
  "inputs": [
    {
@@ -415,6 +490,16 @@
      ],
      "default": "MarketDataPublisher",
    },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
    {
      "id": "dataStreamPublisherEnvironment",
      "type": "pickString",
@@ -1 +1 @@
-PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py
@@ -541,12 +541,11 @@
      "command": "",
      "windows": {
        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
        },
      },
      "icon": {
@@ -561,12 +560,11 @@
      "command": "",
      "windows": {
        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
        },
      },
      "icon": {
@@ -581,12 +579,11 @@
      "command": "",
      "windows": {
        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
+          "cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows",
+          "env": {
+            "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
+          },
        },
      },
      "icon": {
@@ -1048,6 +1045,16 @@
      ],
      "default": "Debug",
    },
+    {
+      "id": "dataStreamQueryAnalyzerEnvironment",
+      "type": "pickString",
+      "description": "DataStreamQueryAnalyzer Configuration",
+      "options": [
+        "Development",
+        "Local",
+      ],
+      "default": "Development",
+    },
    {
      "id": "sparkdCmakePreset",
      "type": "pickString",
@@ -11,9 +11,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{8949AF88-F1D3-4BC4-827F-B088B555F7B9}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{37E4E890-D025-4729-90F9-6F95862F47BC}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{A5FF3FAF-43CB-496B-AF4B-F61EFD645579}"
 EndProject
@@ -97,8 +95,7 @@ Global
 		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{E3BF9514-CB1A-4E07-A8DD-73300FD9726C} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{8949AF88-F1D3-4BC4-827F-B088B555F7B9} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{37E4E890-D025-4729-90F9-6F95862F47BC} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
@@ -43,7 +43,13 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{385C4948-E080-56A6-E29E-8B0F4913175C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{43FB297B-7A4D-4835-A878-E6DF345B580C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -517,6 +523,54 @@ Global
 		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Debug|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x64.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.ActiveCfg = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.Release|x86.Build.0 = Release|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{43FB297B-7A4D-4835-A878-E6DF345B580C}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Debug|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x64.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.ActiveCfg = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.Release|x86.Build.0 = Release|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -535,7 +589,9 @@ Global
 		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
 		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{F3DC52D7-8C59-4EC0-86D9-0E5732EAFCF7} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
@@ -55,14 +55,16 @@ RUN \
   clangd-$clang_major

 # CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
+RUN set -eux \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends ca-certificates curl gpg \
+  && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \
+    | gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+    > /etc/apt/sources.list.d/kitware.list \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends cmake \
+  && rm -rf /var/lib/apt/lists/*

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet8
@@ -80,9 +82,14 @@ RUN mkdir -p /etc/apt/keyrings \
 # Emumba (deps)
 RUN apt-get update && apt-get -y install rapidjson-dev
 # Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
+ARG EMUMBA_DERIBIT_CONNECTOR_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $EMUMBA_DERIBIT_CONNECTOR_DEB_URL -o /tmp/emumba-deribit-connector.deb \
+    && apt-get install -y /tmp/emumba-deribit-connector.deb \
+    && rm -f /tmp/emumba-deribit-connector.deb

 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -90,6 +97,18 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev

+# Rakurai
+ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
+RUN --mount=type=secret,id=bitbucket_access_token \
+  set -eux \
+    && apt-get update \
+    && TOKEN="$(cat /run/secrets/bitbucket_access_token)" \
+    && curl -fL -H "Authorization: Bearer ${TOKEN}" $RAKURAI_DEB_URL -o /tmp/rakurai.deb \
+    # Need --force-overwrite because rakuirai and emumba-deribit-connector both include the same
+    # spdlog headers.
+    && apt-get install -y -o Dpkg::Options::="--force-overwrite" /tmp/rakurai.deb \
+    && rm -f /tmp/rakuirai.deb
+
 # Docker in Docker
 RUN apt-get update && apt-get install -y \
    docker.io \
@@ -18,10 +18,10 @@ MarkupSafe==3.0.2
 mccabe==0.6.1
 numpy==2.2.6
 platformdirs==2.3.0
-protobuf==4.21.12
+protobuf==6.33.4
 pybars3==0.9.7
-pybind11==2.13.6
-pybind11-stubgen==2.5.3
+pybind11==3.0.1
+pybind11-stubgen==2.5.5
 pybit==5.11.0
 pycodestyle==2.7.0
 pycparser==2.20
@@ -31,9 +31,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -387,14 +391,38 @@ Global
 		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
-		{90A48718-F6A7-4747-97EA-89632B451008} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
@@ -0,0 +1,3 @@
+{
+   "dotnet.preferCSharpExtension": true
+}
@@ -0,0 +1 @@
+
@@ -0,0 +1,168 @@
+"""
+Simple DSPublisher DSL websocket client.
+
+Example:
+   python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
+   python dsl_websocket_client.py --dsl-file query.dsl
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+
+DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
+DEFAULT_URL = "ws://localhost:19093/ws/"
+
+
+def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
+   if dsl and dsl_file:
+      raise ValueError("Pass either --dsl or --dsl-file, not both.")
+
+   if dsl_file:
+      return Path(dsl_file).read_text(encoding="utf-8").strip()
+
+   if dsl:
+      return dsl.strip()
+
+   if DEFAULT_DSL.startswith("#"):
+      raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
+
+   return DEFAULT_DSL
+
+
+def load_config_path(config_path: str | None) -> Path | None:
+   if config_path:
+      return Path(config_path)
+
+   environment = os.environ.get("DOTNET_ENVIRONMENT")
+   config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
+   script_dir = Path(__file__).resolve().parent
+   candidate_dirs = [
+      script_dir,
+      script_dir.parent,
+      Path.cwd(),
+   ]
+
+   for candidate_dir in candidate_dirs:
+      candidate_path = candidate_dir / config_file_name
+      if candidate_path.exists():
+         return candidate_path
+
+   return None
+
+
+def load_url(url: str | None, config_path: str | None) -> str:
+   if url:
+      return url
+
+   resolved_config_path = load_config_path(config_path)
+   if not resolved_config_path:
+      print(f"Config file not found; using default url: {DEFAULT_URL}")
+      return DEFAULT_URL
+
+   config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
+   dsl_api_server_config = config.get("DSLAPIServer", {})
+   ip = dsl_api_server_config.get("Ip", "localhost")
+   port = int(dsl_api_server_config.get("Port", 19093))
+   ws_url = f"ws://{ip}:{port}/ws/"
+   print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
+   return ws_url
+
+
+async def send_subscription(websocket, dsl: str) -> None:
+   payload = {
+      "type": "subscribe",
+      "dsl": dsl,
+   }
+   await websocket.send(json.dumps(payload))
+   print("Sent subscribe request:")
+   print(json.dumps(payload, indent=2))
+
+
+async def send_unsubscribe(websocket, subscription_id: str) -> None:
+   payload = {
+      "type": "unsubscribe",
+      "subscriptionId": subscription_id,
+   }
+   await websocket.send(json.dumps(payload))
+   print("Sent unsubscribe request:")
+   print(json.dumps(payload, indent=2))
+
+
+def print_server_message(raw_message: str) -> str | None:
+   try:
+      message = json.loads(raw_message)
+   except json.JSONDecodeError:
+      print(raw_message)
+      return None
+
+   print("Received server message:")
+   print(json.dumps(message, indent=2))
+   return message.get("subscriptionId")
+
+
+async def run_client(url: str, dsl: str) -> None:
+   subscription_id: str | None = None
+
+   async with websockets.connect(url) as websocket:
+      print(f"Connected to DSPublisher DSL websocket: {url}")
+      await send_subscription(websocket, dsl)
+
+      try:
+         async for raw_message in websocket:
+            latest_subscription_id = print_server_message(raw_message)
+            subscription_id = subscription_id or latest_subscription_id
+      finally:
+         if subscription_id:
+            await send_unsubscribe(websocket, subscription_id)
+
+
+def main() -> int:
+   parser = argparse.ArgumentParser(
+      description="Subscribe to a DSPublisher DSL websocket query and print responses."
+   )
+   parser.add_argument(
+      "--url",
+      help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
+   )
+   parser.add_argument(
+      "--config",
+      help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
+   )
+   parser.add_argument(
+      "--dsl",
+      help="DSL expression to subscribe to.",
+   )
+   parser.add_argument(
+      "--dsl-file",
+      help="Path to a text file containing the DSL expression.",
+   )
+   args = parser.parse_args()
+
+   try:
+      dsl = load_dsl(args.dsl, args.dsl_file)
+      url = load_url(args.url, args.config)
+      asyncio.run(run_client(url, dsl))
+      return 0
+   except KeyboardInterrupt:
+      print("stopped")
+      return 0
+   except Exception as ex:
+      print(f"error: {ex}")
+      return 1
+
+
+if __name__ == "__main__":
+   raise SystemExit(main())
@@ -0,0 +1,264 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Net;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+using BCG;
+using Bts.DataStreamCore;
+
+/// <summary>
+/// Websocket-facing API boundary for DSL clients. It opens the listener port,
+/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
+/// Instantiated by DSPAPIServer.
+/// </summary>
+public sealed class DSLAPIWSListener : IDisposable {
+   readonly HttpListener listener = new();
+   readonly IDSLBookkeeper bookkeeper;
+   readonly string prefix;
+   readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
+   CancellationTokenSource? cancellation_token_source;
+   Task? listener_task;
+   bool disposed;
+
+   sealed class ClientConnection {
+      public required string client_id { get; init; }
+      public required WebSocket web_socket { get; init; }
+      public HashSet<Guid> subscription_ids { get; } = new();
+      public SemaphoreSlim send_lock { get; } = new(1, 1);
+      public DataPacketJsonSerializer serializer { get; } = new();
+   }
+
+   public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
+      ArgumentNullException.ThrowIfNull(bookkeeper);
+      this.prefix = prefix;
+      this.bookkeeper = bookkeeper;
+   }
+
+   public Task StartAsync(CancellationToken cancellation_token = default) {
+      _throw_if_disposed();
+      if (listener_task is not null) {
+         return Task.CompletedTask;
+      }
+
+      cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
+      listener.Prefixes.Add(prefix);
+      listener.Start();
+      listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
+      return Task.CompletedTask;
+   }
+
+   public async Task StopAsync() {
+      if (cancellation_token_source is null) {
+         return;
+      }
+
+      cancellation_token_source.Cancel();
+      if (listener.IsListening) {
+         listener.Stop();
+      }
+
+      foreach (var client_connection in clients_by_id.Values) {
+         await _cleanup_client(client_connection);
+      }
+
+      if (listener_task is not null) {
+         await listener_task;
+         listener_task = null;
+      }
+
+      cancellation_token_source.Dispose();
+      cancellation_token_source = null;
+   }
+
+   public void Dispose() {
+      if (disposed) {
+         return;
+      }
+
+      StopAsync().GetAwaiter().GetResult();
+      listener.Close();
+      disposed = true;
+   }
+
+   async Task _listen_loop(CancellationToken cancellation_token) {
+      while (!cancellation_token.IsCancellationRequested) {
+         HttpListenerContext context;
+         try {
+            context = await listener.GetContextAsync();
+         } catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
+            break;
+         } catch (ObjectDisposedException) {
+            break;
+         }
+
+         _ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
+      }
+   }
+
+   async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
+      if (!context.Request.IsWebSocketRequest) {
+         context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
+         context.Response.Close();
+         return;
+      }
+
+      HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
+      var client_connection = new ClientConnection {
+         client_id = Guid.NewGuid().ToString("N"),
+         web_socket = web_socket_context.WebSocket,
+      };
+
+      clients_by_id[client_connection.client_id] = client_connection;
+      try {
+         await _read_messages(client_connection, cancellation_token);
+      } finally {
+         await _cleanup_client(client_connection);
+      }
+   }
+
+   async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
+      var buffer = new byte[64 * 1024];
+      while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
+         var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
+         if (result.MessageType == WebSocketMessageType.Close) {
+            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
+            break;
+         }
+
+         var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
+         await _handle_message(client_connection, json, cancellation_token);
+      }
+   }
+
+   async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
+      DSLWebSocketRequest? request;
+      try {
+         request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
+      } catch (JsonException ex) {
+         await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
+         return;
+      }
+
+      if (request?.type is null) {
+         await _send_error(client_connection, "Request type is required.", cancellation_token);
+         return;
+      }
+
+      if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
+         await _handle_subscribe(client_connection, request, cancellation_token);
+         return;
+      }
+
+      if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
+         await _handle_unsubscribe(client_connection, request, cancellation_token);
+         return;
+      }
+
+      await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
+   }
+
+   async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
+      if (string.IsNullOrWhiteSpace(request.dsl)) {
+         await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
+         return;
+      }
+
+      Guid subscription_id = Guid.Empty;
+      subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
+         if (subscription_id == Guid.Empty) {
+            return;
+         }
+         _ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
+      }, client_connection.client_id);
+
+      client_connection.subscription_ids.Add(subscription_id);
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "subscribed",
+         subscription_id = subscription_id,
+      }, cancellation_token);
+   }
+
+   async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
+      if (request.subscription_id is null) {
+         await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
+         return;
+      }
+
+      var subscription_id = request.subscription_id.Value;
+      if (!client_connection.subscription_ids.Remove(subscription_id)) {
+         await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
+         return;
+      }
+
+      bookkeeper.Unsubscribe(subscription_id);
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "unsubscribed",
+         subscription_id = subscription_id,
+      }, cancellation_token);
+   }
+
+   async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
+      var packet_json = client_connection.serializer.Serialize(packet);
+      using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
+      await _send_response(client_connection, new DSLWebSocketResponse {
+         type = "data",
+         subscription_id = subscription_id,
+         packet = packet_document.RootElement.Clone(),
+      }, cancellation_token);
+   }
+
+   Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
+      _send_response(client_connection, new DSLWebSocketResponse {
+         type = "error",
+         message = message,
+      }, cancellation_token);
+
+   async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
+      if (client_connection.web_socket.State != WebSocketState.Open) {
+         bookkeeper.DisposeClient(client_connection.client_id);
+         return;
+      }
+
+      var json = JsonSerializer.Serialize(response);
+      var bytes = Encoding.UTF8.GetBytes(json);
+      await client_connection.send_lock.WaitAsync(cancellation_token);
+      try {
+         await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
+      } catch {
+         bookkeeper.DisposeClient(client_connection.client_id);
+         throw;
+      } finally {
+         client_connection.send_lock.Release();
+      }
+   }
+
+   async Task _cleanup_client(ClientConnection client_connection) {
+      if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
+         bookkeeper.DisposeClient(client_connection.client_id);
+      }
+
+      if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
+         try {
+            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
+         } catch {
+         }
+      }
+
+      client_connection.send_lock.Dispose();
+      client_connection.web_socket.Dispose();
+   }
+
+   void _throw_if_disposed() {
+      if (disposed) {
+         throw new ObjectDisposedException(nameof(DSLAPIWSListener));
+      }
+   }
+}
@@ -0,0 +1,216 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Collections.Generic;
+
+using BCG;
+using Bts.DataStreamCore;
+using bts.BlueOrb;
+using bts;
+
+/// <summary>
+/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
+/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
+/// Instantiated by DSPAPIServer.
+/// </summary>
+public sealed class DSLBookkeeper : IDSLBookkeeper {
+   sealed class QueryState {
+      public required string dsl { get; init; }
+      public required DSLStream stream { get; init; }
+      public DataStream.ChangeDelegate? stream_handler { get; set; }
+      public HashSet<Guid> subscription_ids { get; } = new();
+      public Queue<DataPacket> pending_packets { get; } = new();
+   }
+
+   sealed class SubscriptionState {
+      public required Guid subscription_id { get; init; }
+      public required string client_id { get; init; }
+      public required string dsl_key { get; init; }
+      public required Action<DataPacket> on_data { get; init; }
+   }
+
+   readonly object lock_obj = new();
+   readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
+   readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
+   readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
+   bool disposed;
+
+   public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
+      ArgumentNullException.ThrowIfNull(on_data);
+      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
+
+      lock (lock_obj) {
+         _throw_if_disposed();
+
+         var dsl_key = _normalize_dsl(dsl);
+         if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
+            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
+            query_state = _create_query_state(dsl_key);
+            queries_by_dsl.Add(dsl_key, query_state);
+         } else {
+            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
+         }
+
+         var subscription_id = Guid.NewGuid();
+         var subscription_state = new SubscriptionState {
+            subscription_id = subscription_id,
+            client_id = client_id,
+            dsl_key = dsl_key,
+            on_data = on_data,
+         };
+
+         subscriptions_by_id.Add(subscription_id, subscription_state);
+         query_state.subscription_ids.Add(subscription_id);
+
+         if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
+            client_subscriptions = new();
+            subscriptions_by_client.Add(client_id, client_subscriptions);
+         }
+         client_subscriptions.Add(subscription_id);
+
+         _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
+         return subscription_id;
+      }
+   }
+
+   public void Unsubscribe(Guid subscription_id) {
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+         _unsubscribe_locked(subscription_id);
+      }
+   }
+
+   public void DisposeClient(string client_id) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
+
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
+            return;
+         }
+
+         foreach (var subscription_id in subscription_ids.ToArray()) {
+            _unsubscribe_locked(subscription_id);
+         }
+      }
+   }
+
+   public void DoWork() {
+      List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
+
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         foreach (var query_state in queries_by_dsl.Values) {
+            while (query_state.pending_packets.TryDequeue(out var packet)) {
+               foreach (var subscription_id in query_state.subscription_ids) {
+                  if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
+                     callbacks.Add((subscription_state.on_data, packet));
+                  }
+               }
+            }
+         }
+      }
+
+      foreach (var callback in callbacks) {
+         callback.on_data(callback.packet);
+      }
+   }
+
+   public void Dispose() {
+      lock (lock_obj) {
+         if (disposed) {
+            return;
+         }
+
+         foreach (var query_state in queries_by_dsl.Values) {
+            _dispose_query_state(query_state);
+         }
+
+         queries_by_dsl.Clear();
+         subscriptions_by_id.Clear();
+         subscriptions_by_client.Clear();
+         disposed = true;
+      }
+   }
+
+   QueryState _create_query_state(string dsl_key) {
+      _log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
+      var stream = new DSLStream(dsl_key);
+      var query_state = new QueryState {
+         dsl = dsl_key,
+         stream = stream,
+      };
+
+      DataStream.ChangeDelegate stream_handler = new_value => {
+         if (new_value is not DataPacket packet) {
+            return;
+         }
+
+         lock (lock_obj) {
+            if (!disposed) {
+               query_state.pending_packets.Enqueue(packet);
+            }
+         }
+      };
+
+      query_state.stream_handler = stream_handler;
+      stream.onChange += stream_handler;
+      return query_state;
+   }
+
+   void _unsubscribe_locked(Guid subscription_id) {
+      if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
+         return;
+      }
+
+      if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
+         client_subscriptions.Remove(subscription_id);
+         if (client_subscriptions.Count == 0) {
+            subscriptions_by_client.Remove(subscription_state.client_id);
+         }
+      }
+
+      if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
+         query_state.subscription_ids.Remove(subscription_id);
+         _log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
+         if (query_state.subscription_ids.Count == 0) {
+            _log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
+            _dispose_query_state(query_state);
+            queries_by_dsl.Remove(subscription_state.dsl_key);
+         }
+      }
+   }
+
+   static string _normalize_dsl(string dsl) =>
+      dsl.Trim();
+
+   static void _dispose_query_state(QueryState query_state) {
+      _log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
+      if (query_state.stream_handler is not null) {
+         query_state.stream.onChange -= query_state.stream_handler;
+         query_state.stream_handler = null;
+      }
+      query_state.subscription_ids.Clear();
+      query_state.pending_packets.Clear();
+   }
+
+   static void _log_lifecycle(string message) {
+      Console.WriteLine(message);
+      Log.Gen.info(message);
+   }
+
+   void _throw_if_disposed() {
+      if (disposed) {
+         throw new ObjectDisposedException(nameof(DSLBookkeeper));
+      }
+   }
+}
@@ -0,0 +1,39 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// Client-to-server websocket protocol message for subscribing and unsubscribing.
+/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
+/// </summary>
+public sealed class DSLWebSocketRequest {
+   [JsonPropertyName("type")]
+   public string? type { get; set; }
+
+   [JsonPropertyName("dsl")]
+   public string? dsl { get; set; }
+
+   [JsonPropertyName("subscriptionId")]
+   public Guid? subscription_id { get; set; }
+}
+
+/// <summary>
+/// Server-to-client websocket protocol message. Data responses carry serialized
+/// DataPacket JSON in the packet field.
+/// Instantiated by DSLAPIWSListener before serializing responses to clients.
+/// </summary>
+public sealed class DSLWebSocketResponse {
+   [JsonPropertyName("type")]
+   public string type { get; set; } = "";
+
+   [JsonPropertyName("subscriptionId")]
+   public Guid? subscription_id { get; set; }
+
+   [JsonPropertyName("message")]
+   public string? message { get; set; }
+
+   [JsonPropertyName("packet")]
+   public JsonElement? packet { get; set; }
+}
@@ -0,0 +1,37 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+/// <summary>
+/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
+/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
+/// Instantiated by Program.cs.
+/// </summary>
+public sealed class DSPAPIServer : IDisposable {
+   readonly DSLBookkeeper bookkeeper;
+   readonly DSLAPIWSListener web_socket_listener;
+
+   public DSPAPIServer(global::DSLAPIServerConfig config) {
+      ArgumentNullException.ThrowIfNull(config);
+      bookkeeper = new DSLBookkeeper();
+      var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
+      web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
+      Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
+   }
+
+   public Task StartAsync(CancellationToken cancellation_token = default) =>
+      web_socket_listener.StartAsync(cancellation_token);
+
+   public Task StopAsync() =>
+      web_socket_listener.StopAsync();
+
+   public void DoWork() =>
+      bookkeeper.DoWork();
+
+   public void Dispose() {
+      web_socket_listener.Dispose();
+      bookkeeper.Dispose();
+   }
+}
@@ -0,0 +1,18 @@
+namespace DSPublisher.DSLWSServer;
+
+using System;
+
+using BCG;
+using Bts.DataStreamCore;
+
+/// <summary>
+/// Internal contract between the websocket listener and the DSL subscription owner.
+/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
+/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
+/// </summary>
+public interface IDSLBookkeeper : IDisposable {
+   Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
+   void Unsubscribe(Guid subscription_id);
+   void DisposeClient(string client_id);
+   void DoWork();
+}
@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings*.json" />
+
+    <None Update="appsettings*.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
+    </None>
+    <None Update="WSPublishers\**\*.py">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="DSLWSClient\**\*.py">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+
+public sealed class DSPublisherConfig {
+   public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
+   public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
+}
+
+public sealed class DSLAPIServerConfig {
+   public string Ip { get; init; } = "localhost";
+   public int Port { get; init; } = 19093;
+}
+
+public sealed class StreamPublishingNode {
+   public string Name { get; init; } = string.Empty;
+   public string Expression { get; init; } = string.Empty;
+}
+
+public sealed class PythonPublishingNode {
+   public string PythonScriptLocation { get; init; } = string.Empty;
+   public string Args { get; init; } = string.Empty;
+}
@@ -0,0 +1,125 @@
+using System;
+using System.IO;
+using System.Text.Json;
+using System.Threading;
+
+using BCG;
+using bts;
+using Bts.PubSub;
+using bts.BlueOrb;
+using DSPublisher.DSLWSServer;
+
+Console.WriteLine("Starting DSPublisher");
+
+var configPath = load_config_path();
+Console.WriteLine($"Loading config from: {configPath}");
+if (!File.Exists(configPath)) {
+   Console.WriteLine($"Config file not found: {configPath}");
+   return;
+}
+
+var json = File.ReadAllText(configPath);
+using var doc = JsonDocument.Parse(json);
+
+if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
+   Console.WriteLine("DSPublisher config section not found");
+   return;
+}
+
+var json_options = new JsonSerializerOptions {
+   PropertyNameCaseInsensitive = true,
+};
+
+var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
+var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
+
+Console.WriteLine("StreamPublishingNodes:");
+foreach (var node in config.StreamPublishingNodes) {
+   Console.WriteLine($"  Name: {node.Name}");
+   Console.WriteLine($"  Expression: {node.Expression}");
+}
+
+Console.WriteLine("PythonPublishingNodes:");
+foreach (var node in config.PythonPublishingNodes) {
+   Console.WriteLine($"  PythonScriptLocation: {node.PythonScriptLocation}");
+   Console.WriteLine($"  Args: {node.Args}");
+}
+
+var pubsub_config = load_pubsub_config(doc, json_options);
+PubSubConfig.SetGlobalConfig(pubsub_config);
+using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
+ViewStreamDatabase.SetGlobalInstance(view_stream_database);
+
+//var pub_table_broker = new PubTableBroker();
+//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
+
+using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
+await dsp_api_server.StartAsync();
+
+while (true) {
+   view_stream_database.doWork();
+   dsp_api_server.DoWork();
+   Thread.Sleep(TimeSpan.FromMilliseconds(10));
+}
+
+string load_config_path() {
+   var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+   var baseFileName = string.IsNullOrWhiteSpace(environment)
+      ? "appsettings.json"
+      : $"appsettings.{environment}.json";
+   var dspPublisherFileName = $"DSPublisher.{baseFileName}";
+   var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
+   if (File.Exists(dspPublisherConfigPath)) {
+      return dspPublisherConfigPath;
+   }
+
+   return Path.Combine(AppContext.BaseDirectory, baseFileName);
+}
+
+PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
+   if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
+      Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
+      return PubSubConfig.Default();
+   }
+
+   var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
+   if (pubsub_config is null) {
+      Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
+      return PubSubConfig.Default();
+   }
+
+   Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
+   return pubsub_config;
+}
+
+DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
+   if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
+      Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
+      return new DSLAPIServerConfig();
+   }
+
+   var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
+   if (dsl_api_server_config is null) {
+      Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
+      return new DSLAPIServerConfig();
+   }
+
+   Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
+   return dsl_api_server_config;
+}
+
+List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
+   var stream_publishers = new List<IDisposable>();
+   foreach (var node in publishing_nodes) {
+      if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
+         Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
+         continue;
+      }
+
+      Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
+      var stream = new DSLStream(node.Expression);
+      stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
+   }
+
+   return stream_publishers;
+}
@@ -0,0 +1 @@
+
@@ -0,0 +1,146 @@
+"""
+Requires:
+   pip install -U databento
+   pip install websockets
+
+Sources:
+   https://databento.com/docs/quickstart/set-up
+   https://databento.com/docs/schemas-and-data-formats/bbo
+"""
+
+import datetime as dt
+import subprocess
+import sys
+import time
+
+try:
+   import databento as db
+except ImportError:
+   print("Installing missing dependency: databento")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
+   import databento as db
+
+from databento.common.error import BentoError
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+from databento_transformer import DatabentoTransformer
+from pubsub_client import PubSubClient
+
+
+API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
+DATASET = "GLBX.MDP3"
+SYMBOL = "ES.FUT"
+PUBSUB_WS_URL = "ws://localhost:19092/ws/"
+BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
+TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
+DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
+
+_pubsub_client = PubSubClient(PUBSUB_WS_URL)
+_transformer = DatabentoTransformer(
+   symbol=SYMBOL,
+   bbo_table_name=BBO_TABLE_NAME,
+   trades_table_name=TRADES_TABLE_NAME,
+   definitions_table_name=DEFINITIONS_TABLE_NAME,
+)
+
+
+def _publish_contract_definitions() -> None:
+   historical_client = db.Historical(API_KEY)
+   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
+   schema_range = dataset_range.get("schema", {}).get("definition", {})
+   available_end = schema_range.get("end", dataset_range["end"])
+
+   if isinstance(available_end, str):
+      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
+   else:
+      available_end_dt = available_end
+
+   # Databento's range end is exclusive. Back off slightly so the request stays
+   # strictly inside the available window.
+   end_dt = available_end_dt - dt.timedelta(seconds=1)
+   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
+   start = start_dt.isoformat()
+   end = end_dt.isoformat()
+
+   definition_data = historical_client.timeseries.get_range(
+      dataset=DATASET,
+      schema="definition",
+      stype_in="parent",
+      symbols=SYMBOL,
+      start=start,
+      end=end,
+   )
+
+   df = definition_data.to_df()
+   payload = _transformer.build_definitions_payload(df)
+   if payload is None:
+      print("No Databento contract definitions returned")
+      return
+
+   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
+   _pubsub_client.publish(payload)
+
+
+def on_record(record) -> None:
+   # Print every callback first so we can see whether Databento is flowing
+   # even if our row extraction logic misses a field shape.
+   print(record)
+
+   payloads = _transformer.transform_record(record)
+   if payloads:
+      for payload in payloads:
+         _pubsub_client.publish(payload)
+      return
+
+   print("Skipping record with no BBO/trade fields")
+
+
+def main() -> None:
+   _pubsub_client.start()
+   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
+   try:
+      _publish_contract_definitions()
+   except Exception as ex:
+      print(f"Contract definition publish failed: {ex}", flush=True)
+
+   while True:
+      try:
+         client = db.Live(key=API_KEY)
+
+         client.subscribe(
+            dataset=DATASET,
+            schema="bbo-1s",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+         client.subscribe(
+            dataset=DATASET,
+            schema="trades",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+
+         client.add_callback(on_record)
+         client.start()
+         print("Databento client started, waiting for BBO and trade records...", flush=True)
+         client.block_for_close()
+         print("Databento client closed cleanly", flush=True)
+         break
+      except BentoError as ex:
+         print(f"Databento live connection error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+      except Exception as ex:
+         print(f"Databento publisher unexpected error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+
+
+if __name__ == "__main__":
+   main()
@@ -0,0 +1,244 @@
+import datetime as dt
+
+
+class DatabentoTransformer:
+   def __init__(
+      self,
+      symbol: str,
+      bbo_table_name: str,
+      trades_table_name: str,
+      definitions_table_name: str,
+   ):
+      self._symbol = symbol
+      self._bbo_table_name = bbo_table_name
+      self._trades_table_name = trades_table_name
+      self._definitions_table_name = definitions_table_name
+      self._instrument_id_to_symbol: dict[int, str] = {}
+      self._last_bbo_row: dict | None = None
+      self._trade_row_id = 0
+      self._trade_table_created = False
+
+   def transform_record(self, record) -> list[dict]:
+      payload = self._handle_bbo_record(record)
+      if payload is not None:
+         return [payload]
+
+      payload = self._handle_trade_record(record)
+      if payload is not None:
+         return [payload]
+
+      return []
+
+   def build_definitions_payload(self, definition_df) -> dict | None:
+      if definition_df is None or definition_df.empty:
+         return None
+
+      df = definition_df.reset_index()
+      if "instrument_class" in df.columns:
+         df = df[df["instrument_class"].astype(str) == "F"]
+      if "security_type" in df.columns:
+         df = df[df["security_type"].astype(str) == "FUT"]
+      if "raw_symbol" in df.columns:
+         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
+         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
+
+      rows: list[dict] = []
+      for idx, row in enumerate(df.itertuples(index=False), start=1):
+         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
+         security_name = str(getattr(row, "raw_symbol", ""))
+         if instrument_id > 0 and security_name:
+            self._instrument_id_to_symbol[instrument_id] = security_name
+
+         rows.append({
+            "id": idx,
+            "security_name": security_name,
+            "instrument_id": instrument_id,
+            "instrument_class": str(getattr(row, "instrument_class", "")),
+            "security_type": str(getattr(row, "security_type", "")),
+            "asset": str(getattr(row, "asset", "")),
+            "exchange": str(getattr(row, "exchange", "")),
+            "currency": str(getattr(row, "currency", "")),
+            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
+            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
+            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
+            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
+            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
+            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
+            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
+            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
+            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
+            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
+            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
+            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
+            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
+            "underlying": str(getattr(row, "underlying", "")),
+         })
+
+      if not rows:
+         return None
+
+      return {
+         "op": "create_table",
+         "table_name": self._definitions_table_name,
+         "rows": rows,
+      }
+
+   def _handle_bbo_record(self, record) -> dict | None:
+      row = self._to_bbo_row(record)
+      if row is None:
+         return None
+
+      if self._last_bbo_row is None:
+         payload = {
+            "op": "create_table",
+            "table_name": self._bbo_table_name,
+            "rows": [row],
+         }
+      else:
+         payload = {
+            "op": "multiple_remove_add",
+            "table_name": self._bbo_table_name,
+            "removed_rows": [self._last_bbo_row],
+            "added_rows": [row],
+         }
+
+      self._last_bbo_row = row
+      return payload
+
+   def _handle_trade_record(self, record) -> dict | None:
+      row = self._to_trade_row(record)
+      if row is None:
+         return None
+
+      row["id"] = self._trade_row_id
+      self._trade_row_id += 1
+
+      if not self._trade_table_created:
+         payload = {
+            "op": "create_table",
+            "table_name": self._trades_table_name,
+            "rows": [row],
+         }
+         self._trade_table_created = True
+         return payload
+
+      return {
+         "op": "multiple_remove_add",
+         "table_name": self._trades_table_name,
+         "removed_rows": [],
+         "added_rows": [row],
+      }
+
+   def _to_bbo_row(self, record) -> dict | None:
+      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
+      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
+      if bid_px is None and ask_px is None:
+         return None
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "bid_px": float(bid_px or 0.0),
+         "ask_px": float(ask_px or 0.0),
+         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
+         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
+      }
+
+   def _to_trade_row(self, record) -> dict | None:
+      price = self._get_attr(record, "price", "px")
+      size = self._get_attr(record, "size")
+      if price is None and size is None:
+         return None
+
+      side = self._get_attr(record, "side", "aggressor_side", default="")
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "price": float(price or 0.0),
+         "size": int(size or 0),
+         "side": str(side),
+      }
+
+   def _get_security_name(self, record) -> str:
+      instrument_id = self._get_attr(record, "instrument_id", default=0)
+      try:
+         instrument_id_int = int(instrument_id or 0)
+      except (TypeError, ValueError):
+         instrument_id_int = 0
+
+      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
+         return self._instrument_id_to_symbol[instrument_id_int]
+
+      value = self._get_attr(
+         record,
+         "symbol",
+         "raw_symbol",
+         "stype_out_symbol",
+         "stype_in_symbol",
+         default=self._symbol,
+      )
+      return str(value)
+
+   def _get_timestamp(self, record) -> int:
+      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
+      try:
+         return int(value)
+      except (TypeError, ValueError):
+         return 0
+
+   def _get_attr(self, record, *names, default=None):
+      for name in names:
+         if hasattr(record, name):
+            value = getattr(record, name)
+            if value is not None:
+               return value
+      return default
+
+   def _get_level_attr(self, record, name, default=None):
+      levels = self._get_attr(record, "levels", default=None)
+      if levels:
+         level0 = levels[0]
+         if hasattr(level0, name):
+            value = getattr(level0, name)
+            if value is not None:
+               return value
+      return default
+
+   def _ns_to_iso(self, timestamp_ns) -> str:
+      if timestamp_ns is None:
+         return ""
+
+      if hasattr(timestamp_ns, "isoformat"):
+         try:
+            if hasattr(timestamp_ns, "to_pydatetime"):
+               timestamp_dt = timestamp_ns.to_pydatetime()
+            else:
+               timestamp_dt = timestamp_ns
+
+            if timestamp_dt is None:
+               return ""
+
+            if getattr(timestamp_dt, "tzinfo", None) is None:
+               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
+            return timestamp_dt.isoformat()
+         except (TypeError, ValueError, AttributeError):
+            pass
+
+      try:
+         ts_int = int(timestamp_ns)
+         if ts_int <= 0:
+            return ""
+         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
+      except (TypeError, ValueError, OSError, OverflowError):
+         return ""
+
+   def _nano_to_float(self, value) -> float:
+      try:
+         return float(value) / 1_000_000_000.0
+      except (TypeError, ValueError):
+         return 0.0
@@ -0,0 +1,58 @@
+import asyncio
+import json
+import queue
+import threading
+
+import websockets
+
+
+class PubSubClient:
+   def __init__(self, ws_url: str):
+      self._ws_url = ws_url
+      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
+      self._sender_thread: threading.Thread | None = None
+
+   def start(self) -> threading.Thread:
+      if self._sender_thread is None or not self._sender_thread.is_alive():
+         self._sender_thread = self._start_sender_thread()
+      return self._sender_thread
+
+   def publish(self, payload: dict) -> None:
+      self._outgoing_queue.put(payload)
+
+   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
+      self.publish({
+         "op": "create_table",
+         "table_name": table_name,
+         "rows": rows,
+      })
+
+   def publish_multiple_remove_add(
+      self,
+      table_name: str,
+      removed_rows: list[dict],
+      added_rows: list[dict],
+   ) -> None:
+      self.publish({
+         "op": "multiple_remove_add",
+         "table_name": table_name,
+         "removed_rows": removed_rows,
+         "added_rows": added_rows,
+      })
+
+   def _start_sender_thread(self) -> threading.Thread:
+      thread = threading.Thread(target=self._run_send_loop, daemon=True)
+      thread.start()
+      return thread
+
+   def _run_send_loop(self) -> None:
+      asyncio.run(self._ws_send_loop())
+
+   async def _ws_send_loop(self) -> None:
+      async with websockets.connect(self._ws_url) as ws:
+         print(f"Connected to PubSubServer websocket: {self._ws_url}")
+         while True:
+            payload = await asyncio.to_thread(self._outgoing_queue.get)
+            body = json.dumps(payload)
+            print(body)
+            await ws.send(body)
@@ -0,0 +1,69 @@
+{
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "DisposedConnectionCheckPeriod": 1000,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "DSLAPIServer": {
+    "Ip": "localhost",
+    "Port": 19093
+  },
+  "DSPublisher": {
+    "StreamPublishingNodes": [
+      {
+        "Name": "sample_stream",
+        "Expression": "MillisecondStream(1000,5)"
+      }
+    ],
+    "PythonPublishingNodes": [
+      {
+        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
+        "Args": ""
+      }
+    ]
+  }
+}
@@ -0,0 +1,69 @@
+{
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "DisposedConnectionCheckPeriod": 1000,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "DSLAPIServer": {
+    "Ip": "localhost",
+    "Port": 19093
+  },
+  "DSPublisher": {
+    "StreamPublishingNodes": [
+      {
+        "Name": "sample_stream",
+        "Expression": "MillisecondStream(1000,5)"
+      }
+    ],
+    "PythonPublishingNodes": [
+      {
+        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
+        "Args": ""
+      }
+    ]
+  }
+}
@@ -2,7 +2,8 @@
   using BCG;

   [DataStreamUserRespresentation("AddColumnsStream")]
-   public class AddColumnsStream: ViewStream {
+   // example: AddColumnsStream(l,"jc1:001")
+   public class AddColumnsStream : ViewStream {
      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
         mySetRealSource(rs);
@@ -0,0 +1,126 @@
+namespace BlueOrb.Operators.Core {
+   using BCG;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+
+   [DataStreamUserRespresentation("HideColumnsStream")]
+   public class HideColumnsStream : ViewStream {
+      // Wrapper row to hide the columns
+      public class MyRow : IBlueRow {
+         #region IBlueRow members
+         private IBlueTableSchema _schema;
+         private IBlueRow _row;
+         public MyRow(IBlueTableSchema schema, IBlueRow row) {
+            _schema = schema;
+            _row = row;
+         }
+         public IBlueTableSchema schema { get { return _schema; } }
+         #endregion
+
+         #region ICollection members
+         public int Count { get { return schema.columnNames.Length; } }
+
+         public bool IsSynchronized { get { return false; } }
+
+         public object SyncRoot { get { return this; } }
+
+         public void CopyTo(Array array, int index) {
+            int i = index;
+            foreach (object o in this as IEnumerable)
+               array.SetValue(o, i++);
+         }
+         #endregion
+
+         #region IDictionary members
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public object this[object key] {
+            get {
+               if (key is string)
+                  return this[(string)key];
+               return this[(int)key];
+            }
+            set { throw new NotSupportedException("The IBlueRow is readonly"); }
+         }
+
+         public object this[int index] {
+            get {
+               return this[schema.columnInfo[index].colName];
+            }
+         }
+
+         public object this[string col_name] {
+            get {
+               if (_schema.contains_column(col_name))
+                  return _row[col_name];
+               else
+                  throw new ApplicationException($"{col_name} not found");
+            }
+         }
+
+         public ICollection Keys { get { return schema.columnNames; } }
+
+         public ICollection Values {
+            get {
+               object[] retVal = new object[schema.columnNames.Length];
+               CopyTo(retVal, 0);
+               return retVal;
+            }
+         }
+
+         public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+         public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+         public bool Contains(object key) { return schema[(string)key] != null; }
+
+         public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+         #endregion
+
+      }
+
+      private readonly ViewStream _parent_str;
+      private readonly HashSet<string> _removed_cols;
+      private BlueTable _state;
+      public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
+         _parent_str = parent_str;
+         _removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
+         if (_removed_cols.Contains("id"))
+            _removed_cols.Remove("id");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _parent_str.onChange += _handle_parent_str_cb;
+      }
+
+      private void _handle_parent_str_cb(object dp_obj) {
+         // this could be made incremental and hence faster
+         DataPacket dp_in = dp_obj as DataPacket;
+         if (_state == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
+               if (!_removed_cols.Contains(ci.colName))
+                  cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
+            }
+            _state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
+         }
+         _state.clear();
+         foreach (IBlueRow r in dp_in.newRows) {
+            _state.addRow(new MyRow(_state.schema, r));
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_state,
+            myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
+         myFireEvent(dp_out);
+      }
+   }
+
+
+}
@@ -0,0 +1,14 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("CartesianProductStream")]
+   public class CartesianProductStream : ViewStream {
+      public CartesianProductStream(ViewStream ls, ViewStream rs) {
+         ls = new AddColumnsStream(ls, "ljc: 001");
+         rs = new AddColumnsStream(rs, "rjc: 001");
+         ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
+         res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
+         mySetRealSource(res);
+      }
+   }
+}
@@ -0,0 +1,15 @@
+namespace BlueOrb.Operators.Core {
+   using System;
+
+   using BCG;
+
+   [DataStreamUserRespresentation("FullJoinStream")]
+   // JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+}
@@ -1,4 +1,5 @@
-namespace bts.BlueOrb {
+namespace bts.BlueOrb
+{
   using BCG;

   using Bts.DataStream;
@@ -13,8 +14,10 @@
   using RowHash = System.Int32;

   [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream {
-      public class JoinBookkeepingData {
+   public class SimpleJoinStream : ViewStream
+   {
+      public class JoinBookkeepingData
+      {
         // initialized in cons
         readonly string[] _join_col_names;
         private readonly BlueRowHasher _hasher;
@@ -23,7 +26,8 @@
         int[]? _join_cols;
         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();

-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
+         {
            _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
            _hasher = row_hasher;
         }
@@ -40,22 +44,28 @@
         public List<IBlueRow>? get_matching_rows(int hash) =>
            _hash_row_map.GetValueOrDefault(hash);

-         private RowHash _get_hash(IBlueRow r) {
-            if (_join_cols == null || _join_cols.Length == 0) {
+         private RowHash _get_hash(IBlueRow r)
+         {
+            if (_join_cols == null || _join_cols.Length == 0)
+            {
               return 0;
            }
            return _hasher.get_hash(r, _join_cols);
         }

-         public void update_data(IDataPacket dp) {
-            if (!ready) {
+         public void update_data(IDataPacket dp)
+         {
+            if (!ready)
+            {
               _schema = dp.newRows.schema;
               _join_cols = _schema.ResolveIndices(_join_col_names);
            }
            _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows) {
+            foreach (IBlueRow r in dp.newRows)
+            {
               int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
+               {
                  hash_rows = new List<IBlueRow>();
                  _hash_row_map.Add(hash, hash_rows);
               }
@@ -64,8 +74,10 @@
         }

         [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready {
-            get {
+         public bool ready
+         {
+            get
+            {
               return _schema != null;
            }
         }
@@ -77,9 +89,11 @@
      IBlueTableSchema? _schema;

      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
+      { }

-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
+      {
         _ls = ls;
         _rs = rs;
         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
@@ -91,30 +105,35 @@
         _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
      }

-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
         _ls.onChange += _ls_onchange;
         _rs.onChange += _rs_onchange;
      }

-      private void _ls_onchange(object dp_obj) {
+      private void _ls_onchange(object dp_obj)
+      {
         var dp = (IDataPacket)dp_obj;
         _ldata.update_data(dp);
         _form_and_fire();
      }

-      private void _rs_onchange(object dp_obj) {
+      private void _rs_onchange(object dp_obj)
+      {
         var dp = (IDataPacket)dp_obj;
         _rdata.update_data(dp);
         _form_and_fire();
      }

-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
         _ls.onChange -= _ls_onchange;
         _rs.onChange -= _rs_onchange;
      }

      [MemberNotNull(nameof(_schema))]
-      private void _build_schema() {
+      private void _build_schema()
+      {
         if (_schema != null)
            return;
         var lschema = Log.notnull(_ldata.schema);
@@ -122,52 +141,64 @@
         var cis = new List<ColumnInfo> {
            new("id", 0, TypeCode.Int32)
         };
-         foreach (ColumnInfo ci in lschema.columnInfo) {
+         foreach (ColumnInfo ci in lschema.columnInfo)
+         {
            string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
         }
-         foreach (ColumnInfo ci in rschema.columnInfo) {
+         foreach (ColumnInfo ci in rschema.columnInfo)
+         {
            string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
         }
         _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
      }

-      private void _form_and_fire() {
+      private void _form_and_fire()
+      {
         if (!_ldata.ready || !_rdata.ready)
            return;
         _build_schema();
         List<object[]> res_rows = new List<object[]>();

         // this is where the join happens
-         foreach (var kv in _ldata.rows) {
+         foreach (var kv in _ldata.rows)
+         {
            RowHash hash = kv.Key;
            List<IBlueRow> l_rows = kv.Value;
            List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);

-            foreach (IBlueRow lr in l_rows) {
-               if (r_rows != null) {
-                  foreach (IBlueRow r in r_rows) {
+            foreach (IBlueRow lr in l_rows)
+            {
+               if (r_rows != null)
+               {
+                  foreach (IBlueRow r in r_rows)
+                  {
                     object[] rowvals = build_row_array(res_rows.Count, lr);
                     copy_row(rowvals, r, false);
                     res_rows.Add(rowvals);
                  }
-               } else {
+               }
+               else
+               {
                  object[] rowvals = build_row_array(res_rows.Count, lr);
                  res_rows.Add(rowvals);
               }
            }
         }
         BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows) {
+         foreach (object[] rval in res_rows)
+         {
            rv.addRow(new BlueRow(_schema, rval));
         }
         var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
         myFireEvent(dp);

-         object[] build_row_array(int id, IBlueRow l_row) {
+         object[] build_row_array(int id, IBlueRow l_row)
+         {
            object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++) {
+            for (int i = 0; i < _schema.columnNames.Length; i++)
+            {
               rv[i] = DBNull.Value;
            }
            rv[0] = id;
@@ -175,30 +206,26 @@
            return rv;
         }

-         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
+         void copy_row(object?[] rval, IBlueRow r, bool is_left)
+         {
            int offset = 1;
            JoinBookkeepingData sdata = _ldata;
-            if (is_left) {
+            if (is_left)
+            {
               offset = 1;
               sdata = _ldata;
-            } else {
+            }
+            else
+            {
               sdata = _rdata;
               offset = 1 + _ldata.RequireSchema().columnInfo.Length;
            }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
+            {
               rval[offset + i] = r[i];
            }
         }
      }
   }

-   [DataStreamUserRespresentation("FullJoinStream")]
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-
 }
@@ -29,22 +29,26 @@ namespace bts.BlueOrb {
         _stream.onChange -= new ChangeDelegate(_onstream_change);
      }

-
+      private bool _orig_schema_has_timestamp = false;
      private void _onstream_change(object o) {
         DataPacket dp = o as DataPacket;
         DateTime now = DateTime.Now;
         if (_table == null) {
            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            if (!schema.contains_column("timestamp")) {
+               _orig_schema_has_timestamp = true;
+               schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            }
            _table = new CircularBufferTable(schema, _size);
         }
         foreach (IBlueRow r in dp.addedRows) {
            MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            ar.setValue("timestamp", now);
+            if (!_orig_schema_has_timestamp)
+               ar.setValue("timestamp", now);
            ar.setValue("id", _id++);
            _table.add_row(ar);
         }
-         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
         myFireEvent(dp_out);
      }
   }
@@ -9,7 +9,7 @@ namespace BCG {
   [DataStreamUserRespresentation("remoteQuery")]
   public class RemoteQueryStream : ViewStream {
      private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = new PublisherCore();
+      private PublisherCore _publisher_core = PublisherCore.Create();
      private ViewStream _source_str;

      public RemoteQueryStream(params string[] lines) {
@@ -1,22 +0,0 @@
-// using System;
-// using BCG;
-// using BCG.Utilities;
-
-// namespace BCG
-// {
-//    public class CartesianProductStream : ViewStream
-//    {
-//       public CartesianProductStream(ViewStream v1, ViewStream v2)
-//       {
-//          mySetRealSource(
-//          new JoinStream (new AddColumnsStream (v1,
-//             "hiddenCartesianColumn=1"),
-//             "hiddenCartesianColumn",
-//             "!id hiddenCartesianColumn dateTime:dateBoundary",
-//             new AddColumnsStream (v2,
-//             "hiddenCartesianColumn=1"),
-//             "hiddenCartesianColumn",
-//             "!id hiddenCartesianColumn"));
-//       }
-//    }
-// }
@@ -70,7 +70,7 @@ namespace bts
        private readonly int _msg_size;
        private readonly TimeSpan _publish_period;
        private DateTime _last_publish_time = DateTime.MinValue;
-        private PublisherCore _pub_core = new PublisherCore();
+        private PublisherCore _pub_core = PublisherCore.Create();
        private BlueTable _table;
        private int _n_pubs = 0;
        private readonly int _max_pubs;
@@ -224,4 +224,4 @@ namespace bts
        }

    }
-}
+}
@@ -1,54 +1,41 @@
 using BCG;
 using BCG.Utilities.General;

+using Bts.PubSub;
 using Bts.PubSub.Routing;

 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using bts;
+using BCG;

-namespace bts
-{
+namespace BlueOrb.Pubsub.Diagnostics {
+   // This publishes data to the pubsub server, subscribes, gets it back and records the time it took for the round trip
   [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker
-   {
-      public class PayloadMessage
-      {
+   public class PubsubEchoTestStream : ViewStream, IWorker {
+      public class PayloadMessage {
         public readonly string? msg_id;
         public readonly Int64 timestamp = 0;
         public readonly byte[]? payload = null;

-         public static byte[] create_data(int payload_size, byte payload_byte)
-         {
+         public static byte[] create_data(int payload_size, byte payload_byte) {
            var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++)
-            {
+            for (int i = 0; i < payload_size; i++) {
               rv[i] = payload_byte;
            }
            return rv;
         }

-         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
-         // {
-         //    guid = guid_;
-         //    timestamp = timestamp_;
-         //    payload = new byte[payload_size];
-         //    for (int i = 0; i < payload_size; i++)
-         //    {
-         //       payload[i] = payload_byte;
-         //    }
-         // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
-         {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
            msg_id = guid_;
            timestamp = timestamp_;
            payload = payload_;
         }

         private PayloadMessage() { }
-         public byte[] serialize()
-         {
+         public byte[] serialize() {
            var str = new MemoryStream();
            BinaryWriter bw = new BinaryWriter(str);
            bw.Write(msg_id ?? string.Empty);
@@ -60,9 +47,10 @@ namespace bts
            return bytes;
         }

-         public static PayloadMessage deserialize(byte[] ser_bytes)
-         {
-            var str = new MemoryStream(ser_bytes);
+         public static PayloadMessage deserialize(byte[] ser_bytes) {
+            var str = new MemoryStream(ser_bytes) {
+               Position = 0
+            };
            BinaryReader br = new BinaryReader(str);
            string guid = br.ReadString();
            Int64 timestamp_ = br.ReadInt64();
@@ -84,18 +72,23 @@ namespace bts
      private readonly string _tag_name;
      private readonly int _payload_sz;
      private readonly IPubSubRouter _router;
-      private enum IOState
-      {
+      private enum IOState {
         SEND = 0,
         RECV = 1
      };
      IOState _state = IOState.SEND;

-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
-         _router = new PubSubRouter(new PubSubRouterConfig() {
-            PublisherRoutes = new() { new(server_url, ".*") },
-            SubscriberRoutes = new() { new(server_url, ".*") },
-         });
+      public PubsubEchoTestStream()
+         : this(new PubSubRouter(), TimeSpan.FromSeconds(1), 8, "ECHO_TEST", ViewStreamDatabase.instance) { }
+
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name)
+         : this(server_url, publish_period, payload_size, tag_name, ViewStreamDatabase.instance) { }
+
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+         : this(createRouter(server_url), publish_period, payload_size, tag_name, vsd) { }
+
+      public PubsubEchoTestStream(IPubSubRouter router, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+         _router = router;
         _vsd = vsd;
         _payload_sz = payload_size;
         var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -114,34 +107,36 @@ namespace bts
      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }

-      protected override void myOnFirstSubscriber()
-      {
-         _pub_core = new PublisherCore(_router);
+      protected override void myOnFirstSubscriber() {
+         _pub_core = PublisherCore.Create(_router);
         _sub_core = new SubscriberCore(_router);
         _sub_core.subscribe(_tag_name);
         _vsd.addWorker(this);

      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         _vsd.removeWorker(this);
         Log.notnull(_sub_core).unsubscribe(_tag_name);
      }

-      private void _fire_dp_after_send()
-      {
+      static PubSubRouter createRouter(string serverUrl) =>
+         new(new PubSubRouterConfig() {
+            PublisherRoutes = new() { new(serverUrl, ".*") },
+            SubscriberRoutes = new() { new(serverUrl, ".*") },
+         });
+
+      private void _fire_dp_after_send() {
         Log.notnull(_last_sent_msg);
         // as soon as the data is published, its reflected in the stream state
         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
            DBNull.Value, DBNull.Value);
         _table.addRow(r);
-         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, (myLastValue as IDataPacket)?.newRows);
         myFireEvent(dp_out);
      }

-      private void _fire_dp_after_recv()
-      {
+      private void _fire_dp_after_recv() {
         Log.notnull(_last_recvd_msg);
         Log.notnull(_last_sent_msg);
         // update the last publish row
@@ -152,17 +147,17 @@ namespace bts
         BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
                                 String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
         _table.addRow(nr);
-         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, (myLastValue as IDataPacket)?.newRows);
+         if (dp_out.HasChanges()) {
+            myFireEvent(dp_out);
+         }
      }

-      private string _get_next_id()
-      {
+      private string _get_next_id() {
         return _table.Count.ToString();
      }

-      private void _publish_next_packet()
-      {
+      private void _publish_next_packet() {
         Log.notnull(_pub_core);
         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
@@ -171,13 +166,10 @@ namespace bts
         _last_publish_time = DateTime.Now;
      }

-      public void doWork()
-      {
-         switch (_state)
-         {
+      public void doWork() {
+         switch (_state) {
            case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period)
-               {
+               if (DateTime.Now - _last_publish_time >= _publish_period) {
                  _publish_next_packet();
                  _last_publish_time = DateTime.Now;
                  _fire_dp_after_send();
@@ -189,8 +181,7 @@ namespace bts
               var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
               updates = updates.Where(u => u.tag == _tag_name);
               var recv_msg = find_packet(updates);
-               if (recv_msg != null)
-               {
+               if (recv_msg != null) {
                  _last_recvd_msg = recv_msg;
                  _fire_dp_after_recv();
                  _state = IOState.SEND;
@@ -213,8 +204,7 @@ namespace bts
         }
      }

-      public static long epoch_now()
-      {
+      public static long epoch_now() {
         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
      }

@@ -1,12 +1,17 @@
 namespace bts {
   using BCG;

+   using bts.collections;
+
+   using IronPython.Runtime.Operations;
+
   using ProtoBuf;

   using System;
   using System.Collections;
   using System.Collections.Generic;
   using System.Linq;
+   using System.Runtime.Intrinsics.Arm;

   public class ProjectData {
      // schema from the base stream
@@ -1,4 +1,5 @@
-namespace BCG {
+namespace BlueOrb.PubSub {
+   using BCG;
   // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
   // and hence epublishes it out to the PS
   public class PublisherStream : ViewStream {
@@ -295,7 +295,7 @@ namespace BCG {
      }

      public static DataPacket build_dp_inc(IBlueRowCollection new_rows, IDataPacket last_fired_dp, bool deep_row_comapre = false) =>
-         build_dp_inc(new_rows, last_fired_dp.newRows, deep_row_comapre);
+         build_dp_inc(new_rows, last_fired_dp?.newRows ?? new BlueTable(new_rows.schema), deep_row_comapre);

      public static DataPacket build_dp_inc(IBlueRowCollection new_rows, IBlueRowCollection lastRows, bool deep_row_comapre = false) {
         if (lastRows is null) {
@@ -1847,7 +1847,7 @@ namespace BCG {
         return sb.ToString();
      }

-      public static string id_string(IBlueRowCollection table) {
+      public static string rows_id_string(this IBlueRowCollection table) {
         string rv = String.Empty;
         foreach (IBlueRow r in table)
            rv += $"{r["id"]} ";
@@ -2246,7 +2246,7 @@ namespace BCG {

      public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
         try {
-            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
+            object rv = (r != null && r.schema.contains_column(col_name)) ? r[col_name] : DBNull.Value;
            if (rv == null || rv == DBNull.Value)
               return error_val;
            else
@@ -0,0 +1,142 @@
+namespace BlueOrb.Data {
+   using BCG;
+
+   using bts;
+
+
+   using System;
+   using System.Linq;
+
+   public static class Comparisons {
+      public static int compare_row(this IBlueRow row, IBlueRow other_row, params string[] col_names) {
+         int comp_res = 0;
+         foreach (string col_name in col_names) {
+            object val1 = row[col_name];
+            object val2 = other_row[col_name];
+            comp_res = compare_row_values(val1, val2);
+            if (comp_res != 0)
+               break;
+         }
+         return comp_res;
+      }
+
+      public static void compare_tables_id(this IBlueRowCollection table1,
+                                           IBlueRowCollection table2,
+                                           out IBlueRowCollection rows_only_in_table1,
+                                           out IBlueRowCollection rows_only_in_table2) {
+         Log.assert(table1 != null);
+         table2 = table2 ?? new BlueTable(table1.schema);
+         var schema = get_output_schema(table1, table2);
+         var table1_only = new BlueTable(schema);
+         var table2_only = new BlueTable(schema);
+
+         if (table1 is not null) {
+            foreach (IBlueRow row in table1)
+               if (table2 is null || !table2.containsID(row.get<int>("id")))
+                  table1_only.addRow(row);
+         }
+
+         if (table2 is not null) {
+            foreach (IBlueRow row in table2)
+               if (table1 is null || !table1.containsID(row.get<int>("id")))
+                  table2_only.addRow(row);
+         }
+
+         rows_only_in_table1 = table1_only;
+         rows_only_in_table2 = table2_only;
+      }
+
+      public static void compare_tables_deep(this IBlueRowCollection table1,
+                                             IBlueRowCollection table2,
+                                             out IBlueRowCollection rows_only_in_table1,
+                                             out IBlueRowCollection rows_only_in_table2) {
+         Log.assert(table1 != null);
+         table2 = table2 ?? new BlueTable(table1.schema);
+         var schema = get_output_schema(table1, table2);
+         validate_matching_schemas(table1, table2);
+
+         var table1_only = new BlueTable(schema);
+         var table2_only = new BlueTable(schema);
+         var compare_cols = schema.columnNames.Where(c => c != "id").ToArray();
+
+         if (table1 is not null) {
+            foreach (IBlueRow row in table1) {
+               if (table2 is null || !table2.containsID(row.get<int>("id"))) {
+                  table1_only.addRow(row);
+                  continue;
+               }
+
+               var other_row = table2[row.get<int>("id")];
+               if (row.compare_row(other_row, compare_cols) != 0) {
+                  table1_only.addRow(row);
+                  table2_only.addRow(other_row);
+               }
+            }
+         }
+
+         if (table2 is not null) {
+            foreach (IBlueRow row in table2)
+               if (table1 is null || !table1.containsID(row.get<int>("id")))
+                  table2_only.addRow(row);
+         }
+
+         rows_only_in_table1 = table1_only;
+         rows_only_in_table2 = table2_only;
+      }
+
+      static int compare_row_values(object v1, object v2) {
+         if (is_null(v1))
+            return is_null(v2) ? 0 : -1;
+         if (is_null(v2))
+            return 1;
+         try {
+            var comparable1 = (IComparable)v1;
+            return comparable1.CompareTo(v2);
+         } catch (InvalidCastException) {
+            if (v1 is byte[] bytes1 && v2 is byte[] bytes2)
+               return compare_bytes(bytes1, bytes2) ? 0 : 1;
+            throw;
+         }
+      }
+
+      static bool compare_bytes(byte[] v1, byte[] v2) {
+         if (v1.Length != v2.Length)
+            return false;
+
+         for (int i = 0; i < v1.Length; i++)
+            if (v1[i] != v2[i])
+               return false;
+
+         return true;
+      }
+
+      static bool is_null(object o) {
+         return o == null || o == DBNull.Value;
+      }
+
+      static IBlueTableSchema get_output_schema(IBlueRowCollection table1, IBlueRowCollection table2) {
+         if (table1 is not null)
+            return table1.schema;
+         if (table2 is not null)
+            return table2.schema;
+         throw new ArgumentNullException(nameof(table1), "At least one table must be non-null.");
+      }
+
+      static void validate_matching_schemas(IBlueRowCollection table1, IBlueRowCollection table2) {
+         if (table1 is null || table2 is null)
+            return;
+
+         var schema1 = table1.schema;
+         var schema2 = table2.schema;
+         if (schema1.columnInfo.Length != schema2.columnInfo.Length)
+            throw new ArgumentException("Tables must have matching schemas.");
+
+         for (int i = 0; i < schema1.columnInfo.Length; i++) {
+            var c1 = schema1.columnInfo[i];
+            var c2 = schema2.columnInfo[i];
+            if (c1.colName != c2.colName || c1.colType != c2.colType)
+               throw new ArgumentException("Tables must have matching schemas.");
+         }
+      }
+   }
+}
@@ -0,0 +1,35 @@
+namespace BlueOrb.Data {
+   using BCG;
+
+   using System;
+   using System.Text;
+
+public static class ToStrings {
+      public static string ToDebugString(this IBlueRow row) {
+         StringBuilder sb = new StringBuilder();
+         foreach (ColumnInfo c in row.schema.columnInfo) {
+            if (sb.Length != 0)
+               sb.Append(", ");
+            sb.Append(c.colName);
+            sb.Append(": ");
+            object val = row[c.colName];
+            sb.Append(is_null(val) ? "NULL" : val.ToString());
+         }
+         return sb.ToString();
+      }
+
+      public static string ToDebugString(this IBlueRowCollection rows) {
+         StringBuilder sb = new StringBuilder();
+         foreach (IBlueRow row in rows) {
+            if (sb.Length != 0)
+               sb.AppendLine();
+            sb.Append(row.ToDebugString());
+         }
+         return sb.ToString();
+      }
+
+      static bool is_null(object o) {
+         return o == null || o == DBNull.Value;
+      }
+   }
+}
@@ -6,94 +6,89 @@ namespace BCG {
   using System.Reflection;
   using System.IO;
   using Bts.DataStreamCore;
+   using System.Collections.Generic;
+   using System.Linq;

   public class PrimitiveManager {
      static MarketDataStreamFactory mdStreamFactory;
-      private IDictionary myTypeLookupTable = new Hashtable();
-      private static PrimitiveManager mySelf;
-      private ArrayList myAssemblyNames = new ArrayList();
+      private Dictionary<string, Type> _dslopname_to_type_map = new Dictionary<string, Type>();
+      private static PrimitiveManager _self;
+      private HashSet<string> _loaded_assemblies = new HashSet<string>();
+      private static string[] _allowed_namespaces_matches = new string[] {
+         "blueorb"
+      };

      private PrimitiveManager() {
-         myLoadAllTypes();
+         _load_all_streams();
      }

-      private void myLoadAllTypes() {
-         AppDomain currentDomain = AppDomain.CurrentDomain;
-         currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
-         loadAssembly(this.GetType().Assembly.FullName);
-         var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         foreach (string assemblyName in assemblies) {
-            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
-            string resolvedAssemblyPath = assemblyName;
-            if (!System.IO.Path.IsPathRooted(assemblyName)) {
-               resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
+      private static bool _is_loadable_stream(Type type) {
+         Func<bool> namespace_match = () => {
+            foreach (string ns in _allowed_namespaces_matches) {
+               if (type.Namespace.ToLower().Contains(ns.ToLower()))
+                  return true;
            }
-            loadAssembly(resolvedAssemblyPath);
+            return false;
+         };
+         bool rv = type.IsSubclassOf(typeof(ViewStream))
+                   && type.IsPublic;
+         //rv = rv && (!type.IsInterface && !type.IsAbstract);
+         rv = rv && namespace_match();
+         return rv;
+      }
+
+      private void _load_all_streams() {
+         load_streams_from_assembly(this.GetType().Assembly.FullName);
+         var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
+         foreach (string assembly_name in assemblies) {
+            bts.Log.Gen.info($"PrimitiveManager._load_all_streams assemblyName: {assembly_name}");
+            string resolved_path_name = assembly_name;
+            if (!System.IO.Path.IsPathRooted(assembly_name)) {
+               resolved_path_name = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assembly_name));
+            }
+            load_streams_from_assembly(resolved_path_name);
         }
      }

-      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
-         foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
-            if (args.Name.Equals(a.FullName))
-               return a;
-         return null;
+      public Type get_dsl_op_type(string dsl_op_name) {
+         Type rv;
+         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
+            return null;
+         return rv;
      }

-      /// <summary>
-      /// Loads all the Stream types associated with the assembly
-      /// </summary>
-      /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_blue_orb = (t) => {
-            //if (!assemblyName.Contains("SharedStreams.dll"))
-            //   return true;
-            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
-         };
-         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
-         if (myAssemblyNames.Contains(assemblyName))
+      public void load_streams_from_assembly(string assembly_name) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assembly_name}");
+         if (_loaded_assemblies.Contains(assembly_name))
            return;
-         myAssemblyNames.Add(assemblyName);
-         Assembly loadedAssembly = GetType().Assembly.FullName.Equals(assemblyName) ?
+         _loaded_assemblies.Add(assembly_name);
+         Assembly loaded_assembly = GetType().Assembly.FullName.Equals(assembly_name) ?
            GetType().Assembly :
-            Assembly.LoadFrom(assemblyName);
+            Assembly.LoadFrom(assembly_name);

-         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_blue_orb(type)
-               && !type.IsAbstract
-               && type.IsSubclassOf(typeof(ViewStream))
-               && type.IsPublic) {
-               bool isVisible = true;
-               string userRep = createDefaultName(type.Name);
+         foreach (Type type in loaded_assembly.GetTypes())
+            if (_is_loadable_stream(type)) {
+               bool is_visible = true;
+               string dsl_name = type.Name; //createDefaultName(type.Name);
               foreach (Attribute att in type.GetCustomAttributes(false))
                  if (att is DataStreamUserRespresentation) {
                     DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
-                     isVisible = dsatt.isVisible;
-                     userRep = isVisible ?
-                        dsatt.friendlyName == null ? userRep : dsatt.friendlyName
+                     is_visible = dsatt.isVisible;
+                     dsl_name = is_visible ?
+                        dsatt.friendlyName == null ? dsl_name : dsatt.friendlyName
                        : null;
                     break;
                  }
-               if (isVisible) {
-                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
-                  myTypeLookupTable[userRep] = type;
+               if (is_visible) {
+                  // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  _dslopname_to_type_map[dsl_name] = type;
               }
            }
      }

-      public static string createDefaultName(string typeName) {
-         if (!typeName.EndsWith("Stream"))
-            return typeName;
-         string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
-         string startString = new String(new char[] { retVal[0] }).ToLower();
-         retVal = retVal.Remove(0, 1);
-         retVal = retVal.Insert(0, startString);
-         return retVal;
-      }
-
-

      public static PrimitiveManager instance {
-         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
+         get { return _self = _self == null ? new PrimitiveManager() : _self; }
      }

      public static void SetMarketDataClient(object value) {
@@ -101,31 +96,20 @@ namespace BCG {
         mdStreamFactory = new(value);
      }

-      /// <summary>
-      /// Returns the Type name corresponding to the primitive name
-      /// </summary>
-      /// <param name="friendlyName"></param>
-      /// <returns></returns>
-      public string find(string friendlyName) {
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-         return (myTypeLookupTable[friendlyName] as Type).Name;
+      public string dsl_to_type_name(string dsl_op_name) {
+         Type rv;
+         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
+            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
+         return rv.Name;
      }

-      /// <summary>
-      /// Constructs and returns a ViewStream of type corresponding to the friendly name
-      /// </summary>
-      /// <param name="friendlyName">friendly primitive name </param>
-      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
-      /// is the desired one</param>
-      /// <returns></returns>
      public ViewStream build(string friendlyName, ICollection args) {
         object[] constructorArgs = null;
         if (args != null && args.Count > 0) {
            constructorArgs = new object[args.Count];
            args.CopyTo(constructorArgs, 0);
         }
-         var type = myTypeLookupTable[friendlyName] as Type
+         var type = _dslopname_to_type_map[friendlyName] as Type
            ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
         if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
            if (mdStreamFactory is null) {
@@ -140,30 +124,14 @@ namespace BCG {
            Activator.CreateInstance(type, constructorArgs) as ViewStream;
      }

-      /// <summary>
-      /// Returns the name of all the primitives 
-      /// </summary>
-      /// <returns></returns>
-      public string[] getPrimitiveNames() {
-         string[] retVal = new string[myTypeLookupTable.Count];
-         myTypeLookupTable.Keys.CopyTo(retVal, 0);
-         return retVal;
+      public string[] get_dsl_op_names() {
+         return _dslopname_to_type_map.Keys.ToArray();
      }

-      /// <summary>
-      /// Return the types of all the streams
-      /// </summary>
-      /// <returns></returns>
-      public Type[] getStreamTypes() {
-         Type[] retVal = new Type[myTypeLookupTable.Count];
-         myTypeLookupTable.Values.CopyTo(retVal, 0);
-         return retVal;
-      }

-      public ICollection getConstructorSigns(string primitiveName) {
+      public ICollection get_dsl_op_cons_args(string dsl_op_name) {
         ArrayList retVal = new ArrayList();
-         Log.assert(myTypeLookupTable[primitiveName] != null);
-         Type t = myTypeLookupTable[primitiveName] as Type;
+         Type t = get_dsl_op_type(dsl_op_name);
         foreach (ConstructorInfo ci in t.GetConstructors()) {
            string sign = "";
            sign = t.Name + " (";
@@ -177,9 +145,18 @@ namespace BCG {
         return retVal;
      }

-      public string getAssemblyName(string primitiveName) {
-         Log.assert(myTypeLookupTable[primitiveName] != null);
-         return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
+      public string get_dslop_assembly_name(string dsl_op_name) {
+         Type rv;
+         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
+            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
+         return rv.Assembly.FullName;
      }
+
+      public string get_qualified_op_name(string dsl_op_name) {
+         Type dsl_op_type = get_dsl_op_type(dsl_op_name);
+         Log.assert(dsl_op_type != null);
+         return $"{dsl_op_type.Namespace}.{dsl_op_name}";
+      }
+
   }
 }
@@ -1,15 +1,10 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq.Expressions;
-using System.Xml.Serialization;
-
-using BCG.Utilities;
-
 using bts;

 using Bts.DataStreamCore;

+using System;
+using System.Collections.Generic;
+
 namespace BCG {
   /// <summary>
   /// This class will be used by the clients to publish tabular data to the pubsub
@@ -22,7 +17,7 @@ namespace BCG {
      protected bool myWaitingForFirstOp = false;

      public PubTableBroker()
-         : this(new PublisherCore()) { }
+         : this(PublisherCore.Create()) { }

      public PubTableBroker(IPublisherCore publisher) {
         _publisher_core = publisher;
@@ -3,23 +3,23 @@ using System.IO;
 using System.Collections.Generic;
 using System.Linq;

+using BlueOrb.Data;
 using bts;
+using System.Threading;

-namespace BCG
-{
-   // This class is responsible for mantaining the state fo a table. It gets the
-   // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
+namespace BCG {
+   // This class is responsible for mantaining the state for a table. It gets the
+   // bytes from the pubsubserver via the handle_updates call 
+   // (called by VSD, - VSD is polling the SubscriberCOre for udpates), deserializes,
   // updates the state of the table, and fires the TableUpdateDelegate.
-   public class TableAccumulator2 : ITableAccumulator
-   {
+   public class TableAccumulator2 : ITableAccumulator {
      private readonly string _tag;
      private readonly ISubscriberCore _subscriber_core;
      IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
      private BlueTable _table;
      private TableUpdateDelegate _on_update_delegate;

-      public TableAccumulator2(string tag, ISubscriberCore sub_core)
-      {
+      public TableAccumulator2(string tag, ISubscriberCore sub_core) {
         _tag = tag;
         _subscriber_core = sub_core;
      }
@@ -28,61 +28,54 @@ namespace BCG

      //public bool is_active() { return _on_update_delegate != null; }

-      public event TableUpdateDelegate on_update
-      {
-         add
-         {
+      public event TableUpdateDelegate on_update {
+         add {
            Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
-            if (_on_update_delegate == null)
-            {
+            if (_on_update_delegate == null) {
               //ie first subscription
               Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
               _subscriber_core.subscribe(_tag);
            }
            _on_update_delegate =
               Delegate.Combine(_on_update_delegate, value) as TableUpdateDelegate;
+            // a new subsriber should get a full callback immediatly (new rows, 0 deleted, all added)
            if (_table != null)
               value(_table, _table, new BlueTable(_table.schema));
         }
-         remove
-         {
+         remove {
            Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
-            if (_on_update_delegate != null)
-            {
+            if (_on_update_delegate != null) {
               _on_update_delegate =
                  Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
               int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
               Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
-               if (_on_update_delegate == null)
-               {
+               if (_on_update_delegate == null) {
                  _subscriber_core.unsubscribe(_tag);
-                  // TODO
-                  //_table = null;
+                  _table = null;
               }
            }

         }
      }
      private int _update_no = 0;
-      public void handle_updates(IEnumerable<byte[]> msgs)
-      {
+      public void handle_updates(IEnumerable<byte[]> msgs) {
         if (msgs == null || msgs.Count() == 0)
            return;
         Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
         IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
-         foreach (var msg in msgs)
-         {
-            bts.StreamIOMessage str_msg = _deser.deserialize(msg);
+         int i = 0;
+         foreach (var msg in msgs) {
+            bts.StreamIOMessage new_msg_txt = _deser.deserialize(msg);
            var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
-            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
-            switch (str_msg.op)
-            {
+            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} object_hash: {GetHashCode()} thread_id: {Thread.CurrentThread.ManagedThreadId} msg: {i++ / msgs.Count()} msg_sz: {msg.Length} new_msg_txt: {new_msg_txt} pre_update_count: {state_txt}");
+
+            switch (new_msg_txt.op) {
               case bts.StreamIOMessage.StreamOp.CREATE:
-                  _table = new BlueTable(str_msg.create_message.rows);
+                  _table = new BlueTable(new_msg_txt.create_message.rows);
                  break;
               case bts.StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
-                  _table.deleteRows(str_msg.multiple_remove_add_message.removed_rows);
-                  _table.addRows(str_msg.multiple_remove_add_message.added_rows);
+                  _table.deleteRows(new_msg_txt.multiple_remove_add_message.removed_rows);
+                  _table.addRows(new_msg_txt.multiple_remove_add_message.added_rows);
                  break;
            }
         }
@@ -91,11 +84,16 @@ namespace BCG
         IBlueRowCollection d_rows, a_rows;
         //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
         BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
+         Log.Gen.info($"TableAccumulator2.handle_updates firing update, _table: {_table.rows_id_string()} d_rows: {d_rows.rows_id_string()} a_rows: {a_rows.rows_id_string()}");
+         if (a_rows.Count == 0 && d_rows.Count == 0 && table_0 != null) {
+            Log.Gen.info($"TableAccmulator2.handle_updates ex! _table: {_table.ToDebugString()} table_0: {table_0.ToDebugString()} a_rows: {a_rows.ToDebugString()} d_rows: {d_rows.ToDebugString()}");
+            //same table was pubslished (maybe surpress it on the publisher or pubsubs)
+            return;
+         }
         _on_update_delegate?.Invoke(_table, a_rows, d_rows);
      }

-      public void update(Stream[] streams)
-      {
+      public void update(Stream[] streams) {
      }

      #endregion
@@ -30,7 +30,7 @@ namespace bts {
      }
   }

-   // This class is created by the SingleTagData for stream tag_type. Every pub
+   // This class is created by the SingleTagData for stream tag_type. Every DS tag has a PubsubServerSingleStreamTagHandler

   public class PubsubServerSingleStreamTagHandler : IPubSubServerTagHandler {
      // this is kept around to deserielize the incoming messages and keep the state of the table
@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
   }

   public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
+      string prelude = "new " + PrimitiveManager.instance.dsl_to_type_name(myName) + " (";

      return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
   }
@@ -11,10 +11,20 @@
    <PackageReference Include="CommandLineParser" Version="2.9.1" />
  </ItemGroup>

-  <ItemGroup>
-    <ProjectReference Include="..\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-  </ItemGroup>
+  <Choose>
+    <When Condition="'$(UseProjectRefs)' == 'true'">
+      <ItemGroup>
+        <ProjectReference Include="..\..\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
+        <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <PackageReference Include="Bts.Extensions.Hosting" Version="$(SparkVersion)" />
+        <PackageReference Include="SharedStreams" Version="$(SparkVersion)" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>

  <ItemGroup>
    <ConfigurationProfile Include="appsettings.Development.json" />
@@ -0,0 +1,146 @@
+"""
+Requires:
+   pip install -U databento
+   pip install websockets
+
+Sources:
+   https://databento.com/docs/quickstart/set-up
+   https://databento.com/docs/schemas-and-data-formats/bbo
+"""
+
+import datetime as dt
+import subprocess
+import sys
+import time
+
+try:
+   import databento as db
+except ImportError:
+   print("Installing missing dependency: databento")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
+   import databento as db
+
+from databento.common.error import BentoError
+
+try:
+   import websockets
+except ImportError:
+   print("Installing missing dependency: websockets")
+   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
+   import websockets
+
+from databento_transformer import DatabentoTransformer
+from pubsub_client import PubSubClient
+
+
+API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
+DATASET = "GLBX.MDP3"
+SYMBOL = "ES.FUT"
+PUBSUB_WS_URL = "ws://localhost:19092/ws/"
+BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
+TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
+DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
+
+_pubsub_client = PubSubClient(PUBSUB_WS_URL)
+_transformer = DatabentoTransformer(
+   symbol=SYMBOL,
+   bbo_table_name=BBO_TABLE_NAME,
+   trades_table_name=TRADES_TABLE_NAME,
+   definitions_table_name=DEFINITIONS_TABLE_NAME,
+)
+
+
+def _publish_contract_definitions() -> None:
+   historical_client = db.Historical(API_KEY)
+   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
+   schema_range = dataset_range.get("schema", {}).get("definition", {})
+   available_end = schema_range.get("end", dataset_range["end"])
+
+   if isinstance(available_end, str):
+      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
+   else:
+      available_end_dt = available_end
+
+   # Databento's range end is exclusive. Back off slightly so the request stays
+   # strictly inside the available window.
+   end_dt = available_end_dt - dt.timedelta(seconds=1)
+   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
+   start = start_dt.isoformat()
+   end = end_dt.isoformat()
+
+   definition_data = historical_client.timeseries.get_range(
+      dataset=DATASET,
+      schema="definition",
+      stype_in="parent",
+      symbols=SYMBOL,
+      start=start,
+      end=end,
+   )
+
+   df = definition_data.to_df()
+   payload = _transformer.build_definitions_payload(df)
+   if payload is None:
+      print("No Databento contract definitions returned")
+      return
+
+   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
+   _pubsub_client.publish(payload)
+
+
+def on_record(record) -> None:
+   # Print every callback first so we can see whether Databento is flowing
+   # even if our row extraction logic misses a field shape.
+   print(record)
+
+   payloads = _transformer.transform_record(record)
+   if payloads:
+      for payload in payloads:
+         _pubsub_client.publish(payload)
+      return
+
+   print("Skipping record with no BBO/trade fields")
+
+
+def main() -> None:
+   _pubsub_client.start()
+   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
+   try:
+      _publish_contract_definitions()
+   except Exception as ex:
+      print(f"Contract definition publish failed: {ex}", flush=True)
+
+   while True:
+      try:
+         client = db.Live(key=API_KEY)
+
+         client.subscribe(
+            dataset=DATASET,
+            schema="bbo-1s",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+         client.subscribe(
+            dataset=DATASET,
+            schema="trades",
+            stype_in="parent",
+            symbols=SYMBOL,
+         )
+
+         client.add_callback(on_record)
+         client.start()
+         print("Databento client started, waiting for BBO and trade records...", flush=True)
+         client.block_for_close()
+         print("Databento client closed cleanly", flush=True)
+         break
+      except BentoError as ex:
+         print(f"Databento live connection error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+      except Exception as ex:
+         print(f"Databento publisher unexpected error: {ex}", flush=True)
+         print("Retrying Databento live connection in 5 seconds...", flush=True)
+         time.sleep(5)
+
+
+if __name__ == "__main__":
+   main()
@@ -0,0 +1,244 @@
+import datetime as dt
+
+
+class DatabentoTransformer:
+   def __init__(
+      self,
+      symbol: str,
+      bbo_table_name: str,
+      trades_table_name: str,
+      definitions_table_name: str,
+   ):
+      self._symbol = symbol
+      self._bbo_table_name = bbo_table_name
+      self._trades_table_name = trades_table_name
+      self._definitions_table_name = definitions_table_name
+      self._instrument_id_to_symbol: dict[int, str] = {}
+      self._last_bbo_row: dict | None = None
+      self._trade_row_id = 0
+      self._trade_table_created = False
+
+   def transform_record(self, record) -> list[dict]:
+      payload = self._handle_bbo_record(record)
+      if payload is not None:
+         return [payload]
+
+      payload = self._handle_trade_record(record)
+      if payload is not None:
+         return [payload]
+
+      return []
+
+   def build_definitions_payload(self, definition_df) -> dict | None:
+      if definition_df is None or definition_df.empty:
+         return None
+
+      df = definition_df.reset_index()
+      if "instrument_class" in df.columns:
+         df = df[df["instrument_class"].astype(str) == "F"]
+      if "security_type" in df.columns:
+         df = df[df["security_type"].astype(str) == "FUT"]
+      if "raw_symbol" in df.columns:
+         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
+         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
+
+      rows: list[dict] = []
+      for idx, row in enumerate(df.itertuples(index=False), start=1):
+         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
+         security_name = str(getattr(row, "raw_symbol", ""))
+         if instrument_id > 0 and security_name:
+            self._instrument_id_to_symbol[instrument_id] = security_name
+
+         rows.append({
+            "id": idx,
+            "security_name": security_name,
+            "instrument_id": instrument_id,
+            "instrument_class": str(getattr(row, "instrument_class", "")),
+            "security_type": str(getattr(row, "security_type", "")),
+            "asset": str(getattr(row, "asset", "")),
+            "exchange": str(getattr(row, "exchange", "")),
+            "currency": str(getattr(row, "currency", "")),
+            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
+            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
+            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
+            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
+            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
+            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
+            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
+            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
+            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
+            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
+            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
+            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
+            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
+            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
+            "underlying": str(getattr(row, "underlying", "")),
+         })
+
+      if not rows:
+         return None
+
+      return {
+         "op": "create_table",
+         "table_name": self._definitions_table_name,
+         "rows": rows,
+      }
+
+   def _handle_bbo_record(self, record) -> dict | None:
+      row = self._to_bbo_row(record)
+      if row is None:
+         return None
+
+      if self._last_bbo_row is None:
+         payload = {
+            "op": "create_table",
+            "table_name": self._bbo_table_name,
+            "rows": [row],
+         }
+      else:
+         payload = {
+            "op": "multiple_remove_add",
+            "table_name": self._bbo_table_name,
+            "removed_rows": [self._last_bbo_row],
+            "added_rows": [row],
+         }
+
+      self._last_bbo_row = row
+      return payload
+
+   def _handle_trade_record(self, record) -> dict | None:
+      row = self._to_trade_row(record)
+      if row is None:
+         return None
+
+      row["id"] = self._trade_row_id
+      self._trade_row_id += 1
+
+      if not self._trade_table_created:
+         payload = {
+            "op": "create_table",
+            "table_name": self._trades_table_name,
+            "rows": [row],
+         }
+         self._trade_table_created = True
+         return payload
+
+      return {
+         "op": "multiple_remove_add",
+         "table_name": self._trades_table_name,
+         "removed_rows": [],
+         "added_rows": [row],
+      }
+
+   def _to_bbo_row(self, record) -> dict | None:
+      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
+      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
+      if bid_px is None and ask_px is None:
+         return None
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "bid_px": float(bid_px or 0.0),
+         "ask_px": float(ask_px or 0.0),
+         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
+         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
+      }
+
+   def _to_trade_row(self, record) -> dict | None:
+      price = self._get_attr(record, "price", "px")
+      size = self._get_attr(record, "size")
+      if price is None and size is None:
+         return None
+
+      side = self._get_attr(record, "side", "aggressor_side", default="")
+
+      return {
+         "id": 0,
+         "security_name": self._get_security_name(record),
+         "timestamp": self._get_timestamp(record),
+         "price": float(price or 0.0),
+         "size": int(size or 0),
+         "side": str(side),
+      }
+
+   def _get_security_name(self, record) -> str:
+      instrument_id = self._get_attr(record, "instrument_id", default=0)
+      try:
+         instrument_id_int = int(instrument_id or 0)
+      except (TypeError, ValueError):
+         instrument_id_int = 0
+
+      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
+         return self._instrument_id_to_symbol[instrument_id_int]
+
+      value = self._get_attr(
+         record,
+         "symbol",
+         "raw_symbol",
+         "stype_out_symbol",
+         "stype_in_symbol",
+         default=self._symbol,
+      )
+      return str(value)
+
+   def _get_timestamp(self, record) -> int:
+      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
+      try:
+         return int(value)
+      except (TypeError, ValueError):
+         return 0
+
+   def _get_attr(self, record, *names, default=None):
+      for name in names:
+         if hasattr(record, name):
+            value = getattr(record, name)
+            if value is not None:
+               return value
+      return default
+
+   def _get_level_attr(self, record, name, default=None):
+      levels = self._get_attr(record, "levels", default=None)
+      if levels:
+         level0 = levels[0]
+         if hasattr(level0, name):
+            value = getattr(level0, name)
+            if value is not None:
+               return value
+      return default
+
+   def _ns_to_iso(self, timestamp_ns) -> str:
+      if timestamp_ns is None:
+         return ""
+
+      if hasattr(timestamp_ns, "isoformat"):
+         try:
+            if hasattr(timestamp_ns, "to_pydatetime"):
+               timestamp_dt = timestamp_ns.to_pydatetime()
+            else:
+               timestamp_dt = timestamp_ns
+
+            if timestamp_dt is None:
+               return ""
+
+            if getattr(timestamp_dt, "tzinfo", None) is None:
+               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
+            return timestamp_dt.isoformat()
+         except (TypeError, ValueError, AttributeError):
+            pass
+
+      try:
+         ts_int = int(timestamp_ns)
+         if ts_int <= 0:
+            return ""
+         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
+      except (TypeError, ValueError, OSError, OverflowError):
+         return ""
+
+   def _nano_to_float(self, value) -> float:
+      try:
+         return float(value) / 1_000_000_000.0
+      except (TypeError, ValueError):
+         return 0.0
@@ -0,0 +1,58 @@
+import asyncio
+import json
+import queue
+import threading
+
+import websockets
+
+
+class PubSubClient:
+   def __init__(self, ws_url: str):
+      self._ws_url = ws_url
+      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
+      self._sender_thread: threading.Thread | None = None
+
+   def start(self) -> threading.Thread:
+      if self._sender_thread is None or not self._sender_thread.is_alive():
+         self._sender_thread = self._start_sender_thread()
+      return self._sender_thread
+
+   def publish(self, payload: dict) -> None:
+      self._outgoing_queue.put(payload)
+
+   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
+      self.publish({
+         "op": "create_table",
+         "table_name": table_name,
+         "rows": rows,
+      })
+
+   def publish_multiple_remove_add(
+      self,
+      table_name: str,
+      removed_rows: list[dict],
+      added_rows: list[dict],
+   ) -> None:
+      self.publish({
+         "op": "multiple_remove_add",
+         "table_name": table_name,
+         "removed_rows": removed_rows,
+         "added_rows": added_rows,
+      })
+
+   def _start_sender_thread(self) -> threading.Thread:
+      thread = threading.Thread(target=self._run_send_loop, daemon=True)
+      thread.start()
+      return thread
+
+   def _run_send_loop(self) -> None:
+      asyncio.run(self._ws_send_loop())
+
+   async def _ws_send_loop(self) -> None:
+      async with websockets.connect(self._ws_url) as ws:
+         print(f"Connected to PubSubServer websocket: {self._ws_url}")
+         while True:
+            payload = await asyncio.to_thread(self._outgoing_queue.get)
+            body = json.dumps(payload)
+            print(body)
+            await ws.send(body)
@@ -59,14 +59,9 @@
          "Class": "bts.PubsubServerStreamTagHandler"
        },
        {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
        },
        {
          "TagType": ".*",
@@ -90,4 +85,4 @@
      }
    ]
  }
-}
+}
@@ -16,6 +16,12 @@
    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
    <ProjectReference Include="..\GuiUtilities\GuiUtilities.csproj" />
    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings.json" />
+    <ConfigurationProfile Include="appsettings.*.json" />
  </ItemGroup>

 </Project>
@@ -2,21 +2,6 @@
 {
   public partial class MainForm
   {
-      /// <summary>
-      /// Clean up any resources being used.
-      /// </summary>
-      protected override void Dispose(bool disposing)
-      {
-         if (disposing)
-         {
-            if (components != null)
-            {
-               components.Dispose();
-            }
-         }
-         base.Dispose(disposing);
-      }
-
      #region Windows Form Designer generated code
      /// <summary>
      /// Required method for Designer support - do not modify
@@ -1,22 +1,75 @@
 using bts;

+using Bts.DataStream;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;

+using Spark.Core.Disposables;
+using Spark.Core.Reactive;
+
 using System;
-using System.Collections.Generic;
 using System.Collections;
+using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics;
 using System.Drawing;
 using System.IO;
+using System.Linq;
+using System.Reactive;
+using System.Reactive.Linq;
+using System.Reactive.Subjects;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Windows.Forms;
+using BlueOrb.Pubsub.Diagnostics;

 namespace BCG
 {
   public partial class MainForm : System.Windows.Forms.Form
   {
+      private const string LastQueryRegistryKey = "LastQuery";
+      private const string RecentSavedFilesRegistryKey = "RecentSavedFiles";
+      private const string DateTimeDisplayFormat = "M/d/yyyy h:mm:ss.fff tt";
+      private const int MaxRecentSavedFiles = 10;
+      private static readonly Color QueryEditorDefaultColor = Color.White;
+      private static readonly Color QueryEditorCommentColor = Color.FromArgb(106, 115, 125);
+      private static readonly Color QueryEditorOperatorColor = Color.FromArgb(86, 156, 214);
+      private static readonly Color QueryEditorAssignmentNameColor = Color.FromArgb(78, 201, 176);
+      private static readonly Color GridBackgroundColor = Color.FromArgb(37, 37, 38);
+      private static readonly Color GridAltRowColor = Color.FromArgb(43, 43, 48);
+      private static readonly Color GridHeaderColor = Color.FromArgb(62, 62, 66);
+      private static readonly Color GridGridLineColor = Color.FromArgb(58, 58, 61);
+      private static readonly Color GridSelectionColor = Color.FromArgb(9, 71, 113);
+      private static readonly Color GridSelectionForeColor = Color.White;
+      private static readonly Color GridTimestampColor = Color.FromArgb(206, 215, 222);
+      private static readonly Color GridIdentifierColor = Color.FromArgb(156, 163, 175);
+      private static readonly Color GridIndicatorColor = Color.FromArgb(156, 220, 254);
+
+      class PrimitiveTreeNodeComparer : IComparer
+      {
+         public int Compare(object? x, object? y)
+         {
+            var left = x as TreeNode;
+            var right = y as TreeNode;
+            if (left is null || right is null)
+               return 0;
+
+            string left_key = _sort_key(left);
+            string right_key = _sort_key(right);
+            return StringComparer.OrdinalIgnoreCase.Compare(left_key, right_key);
+         }
+
+         static string _sort_key(TreeNode node)
+         {
+            if (node.Parent is null)
+               return node.Text;
+
+            if (node.Tag is string primitive_name)
+               return primitive_name;
+
+            return node.Text;
+         }
+      }
      // static async Task _run() {
      //    var login = "bts";
      //    var passwd = String.Empty;
@@ -31,52 +84,98 @@ namespace BCG
      private string? myLastFileName;
      private ViewStream? myCurrentStream;
      private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
+      private readonly List<string> _recent_saved_files = new List<string>();
+      private ToolStripSeparator? _recent_saved_files_separator;
      // private string mySite = SitesCore.getMySiteName ();
      private string mySite = "TODO";
      private string myLastClassName = "MyView";
      private ViewStream? myAllTagsStream;
+      IBlueRowCollection? tagRows;
      private string myKQ;
      private TextWriter? myConsoleOutput;
+      IDataPacket? pendingData;
+      IDisposable? titleSub;
+      readonly ToolTip _primitive_tool_tip = new ToolTip();
+      readonly Dictionary<string, string?> _primitive_source_path_cache = new Dictionary<string, string?>();
+      Dictionary<string, string?>? _source_file_index;
+      string? _repo_root_path;
+      char? _last_primitive_search_char;
+      TreeNode? _last_primitive_search_assembly_node;
+      bool _is_applying_query_syntax_highlighting;
+      HashSet<string>? _dsl_operator_names;

      public MainForm()
      {
         //Task.Run(async () => await _run());
         InitializeComponent();
+         _apply_grid_theme();
         // myKQ = ConfigurationRequester.getApplicationQualifier();         
         myKQ = "TODO";
+         var handleCreatedSubject = new Subject<bool>();
+         HandleCreated += (s, e) => handleCreatedSubject.OnNext(true);
+         titleSub = Disposable.Create(
+            handleCreatedSubject.SelectSeries(_ =>
+               Observable.FromEvent(
+                  cb => Invoke(() => titleInvalidated += cb),
+                  cb => Invoke(() => titleInvalidated -= cb))
+               .Prepend(Unit.Default)
+               .SelectSeries(_ => new PubsubRoutesStream().AsObservable().Take(1)))
+               .Subscribe(dp => BeginInvoke(() => refreshTitle(dp.newRows))),
+            handleCreatedSubject);
         mySetText();

         myWindowState = new PersistWindowState();
         myWindowState.parent = this;
         myWindowState.saveState = false;
+         myWindowState.LoadState += myLoadPersistedState;
+         myWindowState.SaveState += mySavePersistedState;

         this.Load += new EventHandler(myLoad);
         this.Closing += new CancelEventHandler(myClosing);
-         myQueryTextBox.TextChanged += (o, e) =>
-         {
-            myQueryTextBox.ForeColor = Color.White;
-         };
+         myQueryTextBox.TextChanged += (o, e) => _apply_query_syntax_highlighting();
      }

+      event Action? titleInvalidated;
+
      public string query_text
      {
         get { return myQueryTextBox.Text; }
-         set { myQueryTextBox.Text = value; }
+         set
+         {
+            myQueryTextBox.Text = value;
+            _apply_query_syntax_highlighting();
+         }
      }
+
+      /// <summary>
+      /// Clean up any resources being used.
+      /// </summary>
+      protected override void Dispose(bool disposing)
+      {
+         if (disposing)
+         {
+            Disposable.Dispose(ref titleSub);
+            components?.Dispose();
+         }
+         base.Dispose(disposing);
+      }
+
      private ViewStream myGetAllTagsStream()
      {
         // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
         //ViewStream vs = new PubSubServerStateStream();
-         ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
+         ViewStream vs = new PubSubServerStateStream(ViewStreamDatabase.instance);
         return new FilterStream(vs, "tag~='DS_'");
         //return new LiteralStream("tag/String", "DS_table1;DS_table2");
      }

      private void mySetText()
      {
-         ViewStream routes = new PubsubRoutesStream();
-         routes.onChange += (o) => { };
-         var rows = routes.rows;
+         titleInvalidated?.Invoke();
+      }
+
+      void refreshTitle(IBlueRowCollection rows)
+      {
         string route_txt = String.Empty;
         foreach (IBlueRow r in rows)
         {
@@ -97,11 +196,11 @@ namespace BCG
      private IDictionary myGetLibraryData()
      {
         SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
+         string[] primitiveNames = PrimitiveManager.instance.get_dsl_op_names();
         foreach (string pName in primitiveNames)
         {
-            string aName = PrimitiveManager.instance.getAssemblyName(pName);
-            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
+            string aName = PrimitiveManager.instance.get_dslop_assembly_name(pName);
+            ICollection cNames = PrimitiveManager.instance.get_dsl_op_cons_args(pName);
            if (!retVal.Contains(aName))
               retVal.Add(aName, new SortedList());
            (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
@@ -113,6 +212,65 @@ namespace BCG
         return retVal;
      }

+      HashSet<string> _get_dsl_operator_names()
+      {
+         if (_dsl_operator_names != null)
+            return _dsl_operator_names;
+
+         _dsl_operator_names = new HashSet<string>(
+            PrimitiveManager.instance.get_dsl_op_names().Cast<string>(),
+            StringComparer.OrdinalIgnoreCase);
+         return _dsl_operator_names;
+      }
+
+      void _apply_query_syntax_highlighting()
+      {
+         if (_is_applying_query_syntax_highlighting || myQueryTextBox.IsDisposed)
+            return;
+
+         _is_applying_query_syntax_highlighting = true;
+         try
+         {
+            int selection_start = myQueryTextBox.SelectionStart;
+            int selection_length = myQueryTextBox.SelectionLength;
+
+            myQueryTextBox.SuspendLayout();
+            myQueryTextBox.SelectAll();
+            myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
+
+            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"^\s*([A-Za-z_]\w*)\s*=", RegexOptions.Multiline))
+            {
+               Group group = match.Groups[1];
+               myQueryTextBox.Select(group.Index, group.Length);
+               myQueryTextBox.SelectionColor = QueryEditorAssignmentNameColor;
+            }
+
+            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"\b([A-Za-z_]\w*)\s*\("))
+            {
+               Group group = match.Groups[1];
+               if (!_get_dsl_operator_names().Contains(group.Value))
+                  continue;
+               myQueryTextBox.Select(group.Index, group.Length);
+               myQueryTextBox.SelectionColor = QueryEditorOperatorColor;
+            }
+
+            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"//.*$", RegexOptions.Multiline))
+            {
+               myQueryTextBox.Select(match.Index, match.Length);
+               myQueryTextBox.SelectionColor = QueryEditorCommentColor;
+            }
+
+            myQueryTextBox.Select(selection_start, selection_length);
+            if (selection_length == 0)
+               myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
+            myQueryTextBox.ResumeLayout();
+         }
+         finally
+         {
+            _is_applying_query_syntax_highlighting = false;
+         }
+      }
+

      private void myLoadAssemblies()
      {
@@ -123,13 +281,18 @@ namespace BCG
            TreeNode assemblyNode = new TreeNode(de.Key as String);
            foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
            {
-               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
+               string primitive_name = innerDe.Key as string;
+               TreeNode primitiveNode = new TreeNode(primitive_name)
+               {
+                  Tag = primitive_name
+               };
               foreach (string con in (innerDe.Value as ICollection))
                  primitiveNode.Nodes.Add(con);
               assemblyNode.Nodes.Add(primitiveNode);
            }
            myTreeView.Nodes.Add(assemblyNode);
         }
+         myTreeView.Sort();
         /*
         myPrimitivesTextBox.Clear ();
         ArrayList al = new ArrayList(PrimitiveManager.instance.getPrimitiveNames());
@@ -144,8 +307,11 @@ namespace BCG

      private void myHandleTableListChange(object o)
      {
-         IBlueRowCollection rows = (o as DataPacket).newRows;
-         myUpdateTableList();
+         BeginInvoke(() =>
+         {
+            tagRows = ((IDataPacket)o).newRows;
+            myUpdateTableList();
+         });
      }


@@ -155,6 +321,19 @@ namespace BCG
         get { return myQueryTextBox.Height; }
         set { myQueryTextBox.Height = value; }
      }
+
+      private void myLoadPersistedState(object sender, BlueRegistry blueReg)
+      {
+         myQueryTextBox.Text = blueReg.getValue(RegistryRoot.HKCU, LastQueryRegistryKey, string.Empty)?.ToString() ?? string.Empty;
+         _load_recent_saved_files(blueReg.getValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Empty)?.ToString());
+         _refresh_recent_saved_files_menu();
+      }
+
+      private void mySavePersistedState(object sender, BlueRegistry blueReg)
+      {
+         blueReg.setValue(RegistryRoot.HKCU, LastQueryRegistryKey, myQueryTextBox.Text ?? string.Empty);
+         blueReg.setValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Join("|", _recent_saved_files));
+      }
      #endregion

      private Stopwatch _stopwatch = new Stopwatch();
@@ -247,6 +426,7 @@ namespace BCG
            }
            myLastFileName = fileName;
            this.Text = fileName;
+            _track_recent_saved_file(fileName);
         }
         catch (Exception exc)
         {
@@ -331,21 +511,144 @@ namespace BCG
         }
      }

+      private void myApplyDateTimeFormatting(IBlueTableSchema schema)
+      {
+         var dateTimeColumns = new HashSet<string>(
+            schema.columnInfo
+               .Where(ci => ci.colType == typeof(DateTime))
+               .Select(ci => ci.colName));
+
+         foreach (DataGridViewColumn column in myDataGrid.Columns)
+         {
+            if (!dateTimeColumns.Contains(column.DataPropertyName) && !dateTimeColumns.Contains(column.Name))
+               continue;
+
+            column.DefaultCellStyle.Format = DateTimeDisplayFormat;
+            column.Width = Math.Max(column.Width, 220);
+         }
+      }
+
+      void _apply_grid_theme()
+      {
+         myDataGrid.BackgroundColor = GridBackgroundColor;
+         myDataGrid.BorderStyle = BorderStyle.None;
+         myDataGrid.EnableHeadersVisualStyles = false;
+         myDataGrid.GridColor = GridGridLineColor;
+         myDataGrid.CellBorderStyle = DataGridViewCellBorderStyle.SingleHorizontal;
+         myDataGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect;
+         myDataGrid.MultiSelect = false;
+         myDataGrid.RowHeadersVisible = false;
+         myDataGrid.ColumnHeadersBorderStyle = DataGridViewHeaderBorderStyle.Single;
+         myDataGrid.RowHeadersBorderStyle = DataGridViewHeaderBorderStyle.None;
+         myDataGrid.ColumnHeadersHeight = 36;
+         myDataGrid.RowTemplate.Height = 24;
+         myDataGrid.Font = new Font("Consolas", 9F, FontStyle.Regular, GraphicsUnit.Point);
+
+         myDataGrid.DefaultCellStyle.BackColor = GridBackgroundColor;
+         myDataGrid.DefaultCellStyle.ForeColor = Color.White;
+         myDataGrid.DefaultCellStyle.SelectionBackColor = GridSelectionColor;
+         myDataGrid.DefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
+         myDataGrid.DefaultCellStyle.Padding = new Padding(4, 1, 4, 1);
+
+         myDataGrid.AlternatingRowsDefaultCellStyle.BackColor = GridAltRowColor;
+         myDataGrid.AlternatingRowsDefaultCellStyle.ForeColor = Color.White;
+         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionBackColor = GridSelectionColor;
+         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
+
+         myDataGrid.ColumnHeadersDefaultCellStyle.BackColor = GridHeaderColor;
+         myDataGrid.ColumnHeadersDefaultCellStyle.ForeColor = Color.White;
+         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionBackColor = GridHeaderColor;
+         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionForeColor = Color.White;
+         myDataGrid.ColumnHeadersDefaultCellStyle.Font = new Font("Consolas", 9F, FontStyle.Bold, GraphicsUnit.Point);
+         myDataGrid.ColumnHeadersDefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
+         myDataGrid.ColumnHeadersDefaultCellStyle.Padding = new Padding(6, 2, 6, 2);
+      }
+
+      void _apply_grid_column_styling(IBlueTableSchema schema)
+      {
+         var column_types = schema.columnInfo.ToDictionary(ci => ci.colName, ci => ci.colType, StringComparer.OrdinalIgnoreCase);
+         foreach (DataGridViewColumn column in myDataGrid.Columns)
+         {
+            if (!column_types.TryGetValue(column.DataPropertyName, out var col_type) &&
+                !column_types.TryGetValue(column.Name, out col_type))
+            {
+               continue;
+            }
+
+            column.SortMode = DataGridViewColumnSortMode.NotSortable;
+
+            if (_is_numeric_type(col_type))
+            {
+               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleRight;
+            }
+            else
+            {
+               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
+            }
+
+            if (col_type == typeof(DateTime))
+            {
+               column.DefaultCellStyle.ForeColor = GridTimestampColor;
+            }
+            else if (string.Equals(column.HeaderText, "id", StringComparison.OrdinalIgnoreCase))
+            {
+               column.DefaultCellStyle.ForeColor = GridIdentifierColor;
+            }
+            else if (_looks_like_indicator(column.HeaderText))
+            {
+               column.DefaultCellStyle.ForeColor = GridIndicatorColor;
+            }
+            else
+            {
+               column.DefaultCellStyle.ForeColor = Color.White;
+            }
+         }
+
+         static bool _is_numeric_type(Type type)
+         {
+            return type == typeof(int) ||
+                   type == typeof(long) ||
+                   type == typeof(float) ||
+                   type == typeof(double) ||
+                   type == typeof(decimal) ||
+                   type == typeof(short) ||
+                   type == typeof(uint) ||
+                   type == typeof(ulong);
+         }
+
+         static bool _looks_like_indicator(string column_name)
+         {
+            return column_name.Contains("rsi", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("macd", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("atr", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("moving_average", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("signal", StringComparison.OrdinalIgnoreCase) ||
+                   column_name.Contains("histogram", StringComparison.OrdinalIgnoreCase);
+         }
+      }
+
      // this is the stream callback
      private void myCallback(object o)
      {
-         _stopwatch.Stop();
-         var timer = Stopwatch.StartNew();
-         DataPacket dp = o as DataPacket;
-         myCrono.Stop();
-         Log.Ui.diag($"Received callback for {dp.schema.tableName}");
-         if (o == null)
+         var dp = (IDataPacket)o;
+         Log.Ui.diag(dp.GetIncrementalIdString());
+         Volatile.Write(ref pendingData, dp);
+      }
+
+      private Stopwatch myCrono = new();
+
+      private DataStreamExpression? myExp;
+
+      private void myTimer_Tick(object sender, System.EventArgs e)
+      {
+         var dp = Interlocked.Exchange(ref pendingData, null);
+         if (dp is null)
         {
-            System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
-                                                   "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
            return;
         }
-         Log.Ui.diag(dp.toIncrementalIDString());
+         _stopwatch.Stop();
+         var timer = Stopwatch.StartNew();
+         myCrono.Stop();
         myToolStripStatusLabelTime.BackColor = Color.Black;
         myToolStripStatusLabelRows.BackColor = Color.Black;
         myToolStripStatusLabelTime.ForeColor = Color.White;
@@ -357,12 +660,16 @@ namespace BCG
         int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;

         myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
+         _apply_grid_theme();
+         myApplyDateTimeFormatting(dp.newRows.schema);
+         _apply_grid_column_styling(dp.newRows.schema);
+         myDataGrid.ClearSelection();
         var force = false;
         if (myChangeStyle && force)
         {
            myChangeStyle = false;
            myDataGrid.Columns.Clear();
-            myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
+            myDataGrid.Columns.AddRange(Array.ConvertAll(dp.Schema.columnInfo, _column_factory));
         }
         myCrono.Restart();
         // myDataGrid.CaptionText = myCrono.time.TotalMilliseconds + " ms ";
@@ -373,35 +680,13 @@ namespace BCG

         var varNames = new List<string>(0);
         var code = myExp?.print(9, varNames);
-         myPrintCode(dp.schema, varNames, code);
+         myPrintCode(dp.Schema, varNames, code);
         myConsoleRichTextBox.Clear();
         // this was was very very slow
         //myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30);
         timer.Stop();
         Log.Gen.info(
-            $"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms");
-      }
-
-      private Stopwatch myCrono = new();
-
-      private DataStreamExpression? myExp;
-
-      private void myTimer_Tick(object sender, System.EventArgs e)
-      {
-         // try 
-         // {
-         ViewStreamDatabase.instance.doWork();
-         MasterWorker.getInstance(false).doWork();
-         //}
-         // catch (Exception exc) {
-         //    Log.Ui.warn(exc.ToString());
-
-         //    myDataGrid.DataSource = null;
-         //    //myTabControl.SelectedIndex = 1;
-         //    myTabControl.SelectedTab = myErrorTabPage;
-         //    myErrorTextBox.Text = exc.Message;
-         //    myErrorTextBox.Text += "\n" + exc.StackTrace;
-         // }
+            $"DSQA MainForm.myCallback table: {dp.Schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms");
      }

      class StringWrapper
@@ -451,7 +736,7 @@ namespace BCG
      private void myMenuItemOpenClick(object sender, System.EventArgs e)
      {
         OpenFileDialog od = new OpenFileDialog();
-         od.Filter = "C# class (*.cs)|*.cs";
+         od.Filter = "All files (*.*)|*.*|C# class (*.cs)|*.cs";

         if (od.ShowDialog() == DialogResult.OK)
            loadFile(od.FileName);
@@ -463,6 +748,7 @@ namespace BCG
            w.Write(myCSharpText.Text);
         myLastFileName = fileName;
         this.Text = fileName;
+         _track_recent_saved_file(fileName);
      }

      private void myMenuItemSaveClick(object sender, System.EventArgs e)
@@ -485,6 +771,75 @@ namespace BCG
            mySave(sd.FileName);
      }

+      private void _load_recent_saved_files(string? serialized_paths)
+      {
+         _recent_saved_files.Clear();
+         if (string.IsNullOrWhiteSpace(serialized_paths))
+            return;
+
+         foreach (string path in serialized_paths.Split('|'))
+         {
+            if (string.IsNullOrWhiteSpace(path))
+               continue;
+            if (_recent_saved_files.Contains(path, StringComparer.OrdinalIgnoreCase))
+               continue;
+            _recent_saved_files.Add(path);
+            if (_recent_saved_files.Count >= MaxRecentSavedFiles)
+               break;
+         }
+      }
+
+      private void _track_recent_saved_file(string file_name)
+      {
+         _recent_saved_files.RemoveAll(path => string.Equals(path, file_name, StringComparison.OrdinalIgnoreCase));
+         _recent_saved_files.Insert(0, file_name);
+         if (_recent_saved_files.Count > MaxRecentSavedFiles)
+            _recent_saved_files.RemoveRange(MaxRecentSavedFiles, _recent_saved_files.Count - MaxRecentSavedFiles);
+         _refresh_recent_saved_files_menu();
+      }
+
+      private void _refresh_recent_saved_files_menu()
+      {
+         var file_menu = myToolStripMenuItemFile.DropDownItems;
+         for (int i = file_menu.Count - 1; i >= 0; i--)
+         {
+            if (file_menu[i] is ToolStripMenuItem menu_item && Equals(menu_item.Tag, "recent-saved-file"))
+               file_menu.RemoveAt(i);
+         }
+
+         if (_recent_saved_files_separator != null)
+         {
+            file_menu.Remove(_recent_saved_files_separator);
+            _recent_saved_files_separator.Dispose();
+            _recent_saved_files_separator = null;
+         }
+
+         if (_recent_saved_files.Count == 0)
+            return;
+
+         int insert_index = file_menu.IndexOf(myToolStripMenuItemExit);
+         if (insert_index < 0)
+            insert_index = file_menu.Count;
+
+         _recent_saved_files_separator = new ToolStripSeparator();
+         file_menu.Insert(insert_index, _recent_saved_files_separator);
+         insert_index++;
+
+         foreach (string path in _recent_saved_files)
+         {
+            var menu_item = new ToolStripMenuItem(Path.GetFileName(path))
+            {
+               Tag = "recent-saved-file",
+               ToolTipText = path,
+               BackColor = Color.FromArgb(36, 36, 49),
+               ForeColor = Color.Lime
+            };
+            menu_item.Click += (_, __) => loadFile(path);
+            file_menu.Insert(insert_index, menu_item);
+            insert_index++;
+         }
+      }
+
      private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
      {
         if (myTableListBox.SelectedItem != null)
@@ -575,6 +930,7 @@ namespace BCG
         }
         Console.WriteLine("Redirecting console ouput to this window...");

+         myTreeView.TreeViewNodeSorter = new PrimitiveTreeNodeComparer();
         myLoadAssemblies();

         // ViewStreamDatabase.instance = new ViewStreamDatabase();
@@ -582,6 +938,9 @@ namespace BCG
         myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
         myTabControl.SelectedIndex = 3;
         myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
+         myTreeView.KeyPress += myHandleTreeViewKeyPress;
+         myTreeView.NodeMouseHover += myHandleTreeNodeMouseHover;
+         myTreeView.MouseLeave += myHandleTreeViewMouseLeave;
         //do this lazily so we don't have to have bps, etc up to run dsqa
         //_library = new LibraryForm(this);
      }
@@ -622,12 +981,138 @@ namespace BCG
         {
            if (myTreeView.SelectedNode.LastNode == null)
            {
-               string sn = myTreeView.SelectedNode.Parent.Text + "()";
+               string primitive_name = myTreeView.SelectedNode.Parent.Tag as string ?? myTreeView.SelectedNode.Parent.Text;
+               string sn = primitive_name + "()";
               myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
            }
         }
      }

+      private void myHandleTreeViewKeyPress(object? sender, KeyPressEventArgs e)
+      {
+         if (!Char.IsLetterOrDigit(e.KeyChar))
+            return;
+
+         char search_char = Char.ToLowerInvariant(e.KeyChar);
+         TreeNode? assembly_node = _get_selected_assembly_node();
+         if (assembly_node is null)
+            return;
+
+         int start_idx = 0;
+         if (_last_primitive_search_char == search_char &&
+             ReferenceEquals(_last_primitive_search_assembly_node, assembly_node) &&
+             myTreeView.SelectedNode?.Parent == assembly_node)
+         {
+            start_idx = myTreeView.SelectedNode.Index + 1;
+         }
+
+         TreeNode? match_node = _find_matching_primitive_node(assembly_node, search_char, start_idx) ??
+                                _find_matching_primitive_node(assembly_node, search_char, 0);
+
+         if (match_node is not null)
+         {
+            myTreeView.SelectedNode = match_node;
+            match_node.EnsureVisible();
+            _last_primitive_search_char = search_char;
+            _last_primitive_search_assembly_node = assembly_node;
+            e.Handled = true;
+         }
+      }
+
+      private TreeNode? _get_selected_assembly_node()
+      {
+         if (myTreeView.SelectedNode is null)
+            return myTreeView.Nodes.Count > 0 ? myTreeView.Nodes[0] : null;
+
+         return myTreeView.SelectedNode.Parent ?? myTreeView.SelectedNode;
+      }
+
+      private TreeNode? _find_matching_primitive_node(TreeNode assembly_node, char search_char, int start_idx)
+      {
+         for (int i = start_idx; i < assembly_node.Nodes.Count; i++)
+         {
+            TreeNode primitive_node = assembly_node.Nodes[i];
+            string primitive_name = primitive_node.Tag as string ?? primitive_node.Text;
+            if (primitive_name.Length > 0 && Char.ToLowerInvariant(primitive_name[0]) == search_char)
+               return primitive_node;
+         }
+         return null;
+      }
+
+      private void myHandleTreeNodeMouseHover(object? sender, TreeNodeMouseHoverEventArgs e)
+      {
+         if (e.Node?.Tag is not string primitive_name)
+         {
+            _primitive_tool_tip.Hide(myTreeView);
+            return;
+         }
+
+         Type? primitive_type = PrimitiveManager.instance.get_dsl_op_type(primitive_name);
+         if (primitive_type is null)
+         {
+            _primitive_tool_tip.Hide(myTreeView);
+            return;
+         }
+
+         string tooltip_text = $"Namespace: {primitive_type.Namespace ?? "(none)"}";
+         string? source_path = _get_source_file_path(primitive_type);
+         if (!String.IsNullOrWhiteSpace(source_path))
+            tooltip_text += $"\nSource: {source_path}";
+
+         Point mouse_point = myTreeView.PointToClient(Cursor.Position);
+         _primitive_tool_tip.Show(tooltip_text, myTreeView, mouse_point.X + 12, mouse_point.Y + 12, 8000);
+      }
+
+      private void myHandleTreeViewMouseLeave(object? sender, EventArgs e)
+      {
+         _primitive_tool_tip.Hide(myTreeView);
+      }
+
+      private string? _get_source_file_path(Type primitive_type)
+      {
+         string cache_key = primitive_type.FullName ?? primitive_type.Name;
+         if (_primitive_source_path_cache.TryGetValue(cache_key, out string? cached_path))
+            return cached_path;
+
+         Dictionary<string, string?> source_file_index = _source_file_index ??= _build_source_file_index();
+         if (!source_file_index.TryGetValue(primitive_type.Name, out string? source_path))
+            return null;
+         _primitive_source_path_cache[cache_key] = source_path;
+         return source_path;
+      }
+
+      private Dictionary<string, string?> _build_source_file_index()
+      {
+         Dictionary<string, string?> source_file_index = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
+         string? repo_root = _repo_root_path ??= _find_repo_root();
+         if (String.IsNullOrWhiteSpace(repo_root))
+            return source_file_index;
+
+         string source_root = Path.Combine(repo_root, "src", "DataStreamSuite");
+         if (!Directory.Exists(source_root))
+            return source_file_index;
+
+         foreach (string file_path in Directory.EnumerateFiles(source_root, "*.cs", SearchOption.AllDirectories))
+         {
+            string type_name = Path.GetFileNameWithoutExtension(file_path);
+            if (!source_file_index.ContainsKey(type_name))
+               source_file_index[type_name] = file_path;
+         }
+         return source_file_index;
+      }
+
+      private string? _find_repo_root()
+      {
+         string? curr_path = AppDomain.CurrentDomain.BaseDirectory;
+         while (!String.IsNullOrWhiteSpace(curr_path))
+         {
+            if (Directory.Exists(Path.Combine(curr_path, "src", "DataStreamSuite")))
+               return curr_path;
+            curr_path = Directory.GetParent(curr_path)?.FullName;
+         }
+         return null;
+      }
+
      private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
      {
         Close();
@@ -657,7 +1142,7 @@ namespace BCG
         DialogResult dr = ofd.ShowDialog(this);
         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
         {
-            PrimitiveManager.instance.loadAssembly(ofd.FileName);
+            PrimitiveManager.instance.load_streams_from_assembly(ofd.FileName);
            myLoadAssemblies();
         }
      }
@@ -684,7 +1169,7 @@ namespace BCG

      private void myUpdateTableList()
      {
-         IBlueRowCollection rows = myAllTagsStream.rows;
+         var rows = tagRows;
         if (rows == null)
            return;

@@ -742,20 +1227,26 @@ namespace BCG

         ViewStream s = new LoaderStream(@"P:\Dropbox\Data\spread_md_esoa_cme");
         s = new MutableAppendStream(s, s);
-         //s = new MutableAppendStream(s, s);
-         List<IBlueRow> rows = new List<IBlueRow>();
-         foreach (IBlueRow r in s.rows)
-            rows.Add(r);
+         s.onChange += OnChange;

-         for (int i = 0; i < 100; i++)
+         void OnChange(object? obj)
         {
-            timer.Restart();
-            rows.Sort(delegate (IBlueRow r1, IBlueRow r2)
+            s.onChange -= OnChange;
+            //s = new MutableAppendStream(s, s);
+            var rowCollection = ((IDataPacket)obj!).newRows;
+            var rows = new List<IBlueRow>(rowCollection.Count);
+            rows.AddRange(rowCollection.Cast<IBlueRow>());
+
+            for (int i = 0; i < 100; i++)
            {
-               return _sort_dicts_index(r1, r2, 5, true);
-            });
-            timer.Stop();
-            Log.Ui.diag($"nrows = {rows.Count} sorting time = {timer.Elapsed.TotalMilliseconds:0.00}ms");
+               timer.Restart();
+               rows.Sort(delegate (IBlueRow r1, IBlueRow r2)
+               {
+                  return _sort_dicts_index(r1, r2, 5, true);
+               });
+               timer.Stop();
+               Log.Ui.diag($"nrows = {rows.Count} sorting time = {timer.Elapsed.TotalMilliseconds:0.00}ms");
+            }
         }
      }

@@ -1,59 +1,56 @@
 using BCG;

-using bts;
-using bts.api.core;
+using Bts.DataStream.Services;
+using Bts.DataStreamCore.Services;
+using Bts.Extensions.Hosting;
+using Bts.PubSub.Services;

-using Bts.DataStreamQueryAnalyzer;
-using Bts.PubSub;
-
-using CommandLine;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;

 using Spark.Api.Services;
 using Spark.Core.Services;

 using System;
+using System.Threading;
 using System.Threading.Tasks;
 using System.Windows.Forms;

 static class Program {
-   [STAThread]
-   static async Task Main(string[] args) {
-      await Parser.Default.ParseArguments<DataStreamQueryAnalyzerConfig>(args)
-         .MapResult(RunAsync, errs => Task.FromResult(1));
-
-      async Task<int> RunAsync(DataStreamQueryAnalyzerConfig cfg) {
-         try {
-            cfg.MergeWithConfigFile();
-            cfg.MergeWith(DataStreamQueryAnalyzerConfig.Default());
-            PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
-            var configLoader = new SparkConfigLoader(new SparkConfigLoaderOpts() { ConfigServer = cfg.Sparkd });
-            var serviceContext = new ServiceContext(configLoader);
-            var credentials = new SparkApiCredentials() {
-               User = cfg.User,
-               Password = cfg.Password
-            };
-            var loginService = new SparkdLoginService(serviceContext, credentials, configLoader);
-            var permissions = new UserDataPermissions(loginService);
-            var mdClientConfig = new MarketDataClientConfig() { HotPollInterval = TimeSpan.FromSeconds(0.5) };
-            var mdClient = new MarketDataClient(configLoader, loginService, permissions, mdClientConfig);
-            PrimitiveManager.SetMarketDataClient(mdClient);
-            using var vsd = new ViewStreamDatabase(new SubscriberCoreFactory());
-            ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(new SubscriberCoreFactory()));
-            Log.Gen.info("Starting DataStreamQueryAnalyzer");
-            var mainForm = new MainForm();
-            if (cfg.StreamFile is not null) {
-               mainForm.loadFile(cfg.StreamFile);
-            }
-            Application.Run(mainForm);
-         } catch (Exception ex) {
-            // Write the error to console first, in case the logging system is compromised.
-            var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
-            Console.Error.WriteLine(error_msg);
-            Console.Error.Flush();
-            bts.Log.Gen.error(error_msg);
-            return 1;
-         }
-         return 0;
+   static async Task<int> Main(string[] args) {
+      Console.Write("Starting DataStreamQueryAnalyzer");
+      int exitCode = 0;
+      IHost? host = null;
+      try {
+         host = HostBuilderExtensions.CreateBtsBuilder(args)
+            .ConfigureServices(ConfigureServices)
+            .Build();
+         await host.StartAsync();
+         var winFormsThread = new Thread(RunWinFormsApp);
+         winFormsThread.SetApartmentState(ApartmentState.STA);
+         winFormsThread.Start();
+         winFormsThread.Join();
+      } catch (Exception ex) {
+         Console.Error.Write($"DataStreamQueryAnalyzer crashed: {ex}");
+         exitCode = 1;
      }
+      if (host is not null) {
+         await host.StopAsync();
+      }
+      return exitCode;
   }
+
+   static void RunWinFormsApp() {
+      var win = new MainForm();
+      Application.Run(win);
+   }
+
+   static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
+      services.AddSpark(context.Configuration.GetSection("Spark"))
+         .AddSparkApi(context.Configuration.GetSection("SparkApi"))
+         .AddMasterWorker(context.Configuration.GetSection("MasterWorker"))
+         .AddViewStreamDatabase()
+         .AddPubSubClient(context.Configuration.GetSection("PubSub"))
+         .AddPubTableBroker()
+         .AddRemoteQueryManager();
 }
@@ -5,10 +5,9 @@
  "ViewStreamDatabase": {
    "Interval": "00:00:00.1"
  },
-  "Spark": {
-    "ConfigServer": "http://127.0.0.1:8000/config"
-  },
+  "Spark": {},
  "SparkApi": {
+    "IsEnabled": false,
    "Credentials": {
      "User": "bts",
      "Password": ""
@@ -53,21 +52,16 @@
          "Class": "bts.PubsubServerStreamTagHandler"
        },
        {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
+          "TagType": "DS_.*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
        },
        {
          "TagType": ".*",
          "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
+          "Class": "bts.PubsubServerGenericDataHandler"
        }
      ]
    }
  }
-}
+}
@@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "PubSub\PubSub.csp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSubServer", "PubSubServer\PubSubServer.csproj", "{0DBD814A-E17F-4836-8EEE-6E91B00C578F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersistenceServer", "PersistenceServer\PersistenceServer.csproj", "{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "BooleanExpressions\BooleanExpressions.csproj", "{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{88E70136-91AE-4F56-B243-8F4642B46D6A}"
@@ -37,9 +39,15 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "..\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "Bts.Net\Bts.Net.csproj", "{496333E0-E11D-45EE-B0D6-BBAA824A120D}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bts.Net.csproj", "{496333E0-E11D-45EE-B0D6-BBAA824A120D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSPublisher", "DSPublisher\DSPublisher.csproj", "{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIBridgeServer", "APIBridgeServer\APIBridgeServer.csproj", "{3B14C417-531A-4E39-80C1-A4C14BD000D5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -111,6 +119,18 @@ Global
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x64.Build.0 = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.ActiveCfg = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.Build.0 = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.Build.0 = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.Build.0 = Debug|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.Build.0 = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.ActiveCfg = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.Build.0 = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.ActiveCfg = Release|Any CPU
+		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.Build.0 = Release|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -279,6 +299,42 @@ Global
 		{496333E0-E11D-45EE-B0D6-BBAA824A120D}.Release|x64.Build.0 = Release|Any CPU
 		{496333E0-E11D-45EE-B0D6-BBAA824A120D}.Release|x86.ActiveCfg = Release|Any CPU
 		{496333E0-E11D-45EE-B0D6-BBAA824A120D}.Release|x86.Build.0 = Release|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Debug|x64.Build.0 = Debug|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Debug|x86.Build.0 = Debug|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.ActiveCfg = Release|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
+		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.Build.0 = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.Build.0 = Debug|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.ActiveCfg = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.Build.0 = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.ActiveCfg = Release|Any CPU
+		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.Build.0 = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.Build.0 = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.Build.0 = Debug|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.ActiveCfg = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.Build.0 = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.ActiveCfg = Release|Any CPU
+		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -27,6 +27,7 @@
    <ProjectReference Include="..\PubSub\PubSub.csproj" />
    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
  </ItemGroup>


@@ -0,0 +1,121 @@
+namespace Bts.DataStreamTests;
+
+using BCG;
+
+using BlueOrb.Operators.Signal;
+
+using Bts.DataStreamCore;
+
+[TestFixture]
+public class HeikinAshiStreamTests {
+   readonly BlueTableSchema schema = new(
+      new ColumnInfo[] {
+         new("id", 0, TypeCode.Int32),
+         new("ts_event", 1, TypeCode.DateTime),
+         new("open", 2, TypeCode.Double),
+         new("high", 3, TypeCode.Double),
+         new("low", 4, TypeCode.Double),
+         new("close", 5, TypeCode.Double),
+         new("volume", 6, TypeCode.Double),
+      }, "bars");
+
+   [Test]
+   public void AppendsHistoryAndCalculatesRecursiveHeikinAshiValues() {
+      var driver = new TestDriverStream(schema, null);
+      var heikinAshi = new HeikinAshiStream(driver);
+      var packets = new List<IDataPacket>();
+      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
+      );
+
+      Assert.That(packets, Has.Count.EqualTo(2));
+
+      Assert.That(packets[0].newRows.Count, Is.EqualTo(1));
+      Assert.That(packets[0].addedRows.Count, Is.EqualTo(1));
+      Assert.That(packets[0].deletedRows.Count, Is.EqualTo(0));
+
+      var first = packets[0].newRows[0];
+      Assert.That(first.get<double>("open"), Is.EqualTo(100.0).Within(1e-9));
+      Assert.That(first.get<double>("high"), Is.EqualTo(110.0).Within(1e-9));
+      Assert.That(first.get<double>("low"), Is.EqualTo(90.0).Within(1e-9));
+      Assert.That(first.get<double>("close"), Is.EqualTo(105.0).Within(1e-9));
+      Assert.That(first.get<double>("ha_open"), Is.EqualTo(102.5).Within(1e-9));
+      Assert.That(first.get<double>("ha_high"), Is.EqualTo(110.0).Within(1e-9));
+      Assert.That(first.get<double>("ha_low"), Is.EqualTo(90.0).Within(1e-9));
+      Assert.That(first.get<double>("ha_close"), Is.EqualTo(101.25).Within(1e-9));
+
+      Assert.That(packets[1].newRows.Count, Is.EqualTo(2));
+      Assert.That(packets[1].addedRows.Count, Is.EqualTo(1));
+      Assert.That(packets[1].deletedRows.Count, Is.EqualTo(0));
+
+      var second = packets[1].newRows[1];
+      Assert.That(second.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
+      Assert.That(second.get<double>("high"), Is.EqualTo(112.0).Within(1e-9));
+      Assert.That(second.get<double>("low"), Is.EqualTo(104.0).Within(1e-9));
+      Assert.That(second.get<double>("close"), Is.EqualTo(110.0).Within(1e-9));
+      Assert.That(second.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(second.get<double>("ha_high"), Is.EqualTo(112.0).Within(1e-9));
+      Assert.That(second.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(second.get<double>("ha_close"), Is.EqualTo(108.0).Within(1e-9));
+      Assert.That(second.get<double>("volume"), Is.EqualTo(20.0).Within(1e-9));
+   }
+
+   [Test]
+   public void RebuildsHistoryWhenSourceSnapshotChanges() {
+      var driver = new TestDriverStream(schema, null);
+      var heikinAshi = new HeikinAshiStream(driver);
+      var packets = new List<IDataPacket>();
+      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
+      );
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 114.0, 103.0, 111.0, 21.0)
+      );
+
+      Assert.That(packets, Has.Count.EqualTo(3));
+
+      Assert.That(packets[2].newRows.Count, Is.EqualTo(2));
+      Assert.That(packets[2].addedRows.Count, Is.EqualTo(1));
+      Assert.That(packets[2].deletedRows.Count, Is.EqualTo(1));
+
+      var updated = packets[2].newRows[1];
+      Assert.That(updated.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
+      Assert.That(updated.get<double>("high"), Is.EqualTo(114.0).Within(1e-9));
+      Assert.That(updated.get<double>("low"), Is.EqualTo(103.0).Within(1e-9));
+      Assert.That(updated.get<double>("close"), Is.EqualTo(111.0).Within(1e-9));
+      Assert.That(updated.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(updated.get<double>("ha_high"), Is.EqualTo(114.0).Within(1e-9));
+      Assert.That(updated.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
+      Assert.That(updated.get<double>("ha_close"), Is.EqualTo(108.5).Within(1e-9));
+      Assert.That(updated.get<double>("volume"), Is.EqualTo(21.0).Within(1e-9));
+   }
+
+   static IBlueRow Row(int id, DateTime tsEvent, double open, double high, double low, double close, double volume) {
+      var row = new MutableBlueRow(
+         new BlueTableSchema(new[] {
+            new ColumnInfo("id", 0, TypeCode.Int32),
+            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
+            new ColumnInfo("open", 2, TypeCode.Double),
+            new ColumnInfo("high", 3, TypeCode.Double),
+            new ColumnInfo("low", 4, TypeCode.Double),
+            new ColumnInfo("close", 5, TypeCode.Double),
+            new ColumnInfo("volume", 6, TypeCode.Double),
+         }, "bars"));
+      row.setValue("id", id);
+      row.setValue("ts_event", tsEvent);
+      row.setValue("open", open);
+      row.setValue("high", high);
+      row.setValue("low", low);
+      row.setValue("close", close);
+      row.setValue("volume", volume);
+      return row;
+   }
+}
@@ -0,0 +1,53 @@
+namespace Bts.DataStreamTests;
+
+using BCG;
+
+using BlueOrb.Operators.Signal;
+
+using Bts.DataStreamCore;
+
+[TestFixture]
+public class VWAPStreamTests {
+   readonly BlueTableSchema schema = new(
+      new ColumnInfo[] {
+         new("id", 0, TypeCode.Int32),
+         new("ts_event", 1, TypeCode.DateTime),
+         new("close", 2, TypeCode.Double),
+         new("volume", 3, TypeCode.Double),
+      }, "bars");
+
+   [Test]
+   public void EmitsRowsAndCalculatesRollingVWAP() {
+      var driver = new TestDriverStream(schema, null);
+      var vwap = new VWAPStream(driver, "close", "volume", 2);
+      var packets = new List<IDataPacket>();
+      vwap.onChange += nv => packets.Add((IDataPacket)nv);
+
+      driver.Reset(
+         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 10.0),
+         Row(2, new DateTime(2026, 1, 1, 9, 30, 5, DateTimeKind.Utc), 110.0, 20.0)
+      );
+
+      Assert.That(packets, Has.Count.EqualTo(1));
+      Assert.That(packets[0].newRows.Count, Is.EqualTo(2));
+
+      var rows = packets[0].newRows.to_list().OrderBy(r => r.get<int>("id")).ToList();
+      Assert.That(rows[0].get<double>("vwap"), Is.EqualTo(100.0).Within(1e-9));
+      Assert.That(rows[1].get<double>("vwap"), Is.EqualTo((100.0 * 10.0 + 110.0 * 20.0) / 30.0).Within(1e-9));
+   }
+
+   static IBlueRow Row(int id, DateTime tsEvent, double close, double volume) {
+      var row = new MutableBlueRow(
+         new BlueTableSchema(new[] {
+            new ColumnInfo("id", 0, TypeCode.Int32),
+            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
+            new ColumnInfo("close", 2, TypeCode.Double),
+            new ColumnInfo("volume", 3, TypeCode.Double),
+         }, "bars"));
+      row.setValue("id", id);
+      row.setValue("ts_event", tsEvent);
+      row.setValue("close", close);
+      row.setValue("volume", volume);
+      return row;
+   }
+}
@@ -0,0 +1,33 @@
+using System;
+using System.IO;
+using System.Text.Json;
+
+public static class ConfigLoader {
+   public static PersistenceServerConfig load() {
+      var config = new PersistenceServerConfig();
+      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.json"));
+      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.Development.json"));
+      return config;
+   }
+
+   private static void merge_from_file(PersistenceServerConfig target, string path) {
+      if (!File.Exists(path)) {
+         return;
+      }
+
+      var json = File.ReadAllText(path);
+      var loaded = JsonSerializer.Deserialize<PersistenceServerConfig>(json, new JsonSerializerOptions {
+         PropertyNameCaseInsensitive = true
+      });
+
+      if (loaded?.pub_sub is null) {
+         return;
+      }
+
+      if (target.pub_sub is null) {
+         target.pub_sub = loaded.pub_sub;
+      } else {
+         target.pub_sub.MergeWith(loaded.pub_sub);
+      }
+   }
+}
@@ -0,0 +1,148 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SQLite;
+using System.IO;
+using System.Linq;
+
+public class PeristenceHelper {
+   private readonly string _connectionString;
+   public string database_path { get; }
+
+   public PeristenceHelper(string? databasePath = null) {
+      var resolvedPath = databasePath ?? Path.Combine(AppContext.BaseDirectory, "PersistenceServer.sqlite");
+      database_path = resolvedPath;
+      Directory.CreateDirectory(Path.GetDirectoryName(resolvedPath)!);
+      _connectionString = $"Data Source={resolvedPath};Version=3";
+   }
+
+   public void InsertRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      ValidateRow(tableName, row);
+      using var connection = OpenConnection();
+      EnsureTable(connection, tableName, row.Keys);
+
+      var columns = row.Keys.Select(QuoteIdentifier).ToArray();
+      var parameterNames = row.Keys.Select(CreateParameterName).ToArray();
+      var commandText =
+         $"INSERT INTO {QuoteIdentifier(tableName)} ({string.Join(", ", columns)}) VALUES ({string.Join(", ", parameterNames)})";
+
+      using var command = new SQLiteCommand(commandText, connection);
+      AddParameters(command, row);
+      command.ExecuteNonQuery();
+   }
+
+   public void UpdateRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      ValidateRow(tableName, row);
+      using var connection = OpenConnection();
+      EnsureTable(connection, tableName, row.Keys);
+
+      var keyColumns = GetKeyColumns(row);
+      var valueColumns = row.Keys.Where(column => !keyColumns.Contains(column, StringComparer.OrdinalIgnoreCase)).ToArray();
+      if (valueColumns.Length == 0) {
+         throw new InvalidOperationException("UpdateRow requires at least one non-key column to update.");
+      }
+
+      var setClause = string.Join(", ", valueColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateParameterName(column)}"));
+      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
+      var commandText = $"UPDATE {QuoteIdentifier(tableName)} SET {setClause} WHERE {whereClause}";
+
+      using var command = new SQLiteCommand(commandText, connection);
+      foreach (var column in valueColumns) {
+         command.Parameters.AddWithValue(CreateParameterName(column), row[column]);
+      }
+
+      foreach (var column in keyColumns) {
+         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
+      }
+
+      command.ExecuteNonQuery();
+   }
+
+   public void RemoveRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      ValidateRow(tableName, row);
+      using var connection = OpenConnection();
+      EnsureTable(connection, tableName, row.Keys);
+
+      var keyColumns = GetKeyColumns(row);
+      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
+      var commandText = $"DELETE FROM {QuoteIdentifier(tableName)} WHERE {whereClause}";
+
+      using var command = new SQLiteCommand(commandText, connection);
+      foreach (var column in keyColumns) {
+         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
+      }
+
+      command.ExecuteNonQuery();
+   }
+
+   private SQLiteConnection OpenConnection() {
+      var connection = new SQLiteConnection(_connectionString);
+      connection.Open();
+      return connection;
+   }
+
+   private static void ValidateRow(string tableName, IReadOnlyDictionary<string, string> row) {
+      if (string.IsNullOrWhiteSpace(tableName)) {
+         throw new ArgumentException("Table name is required.", nameof(tableName));
+      }
+
+      if (row.Count == 0) {
+         throw new ArgumentException("Row must contain at least one column.", nameof(row));
+      }
+   }
+
+   private static string[] GetKeyColumns(IReadOnlyDictionary<string, string> row) {
+      if (row.ContainsKey("namespace") && row.ContainsKey("key")) {
+         return new[] { "namespace", "key" };
+      }
+
+      if (row.ContainsKey("key")) {
+         return new[] { "key" };
+      }
+
+      throw new InvalidOperationException("Row must contain 'key', or both 'namespace' and 'key', for update/delete.");
+   }
+
+   private static void EnsureTable(SQLiteConnection connection, string tableName, IEnumerable<string> columns) {
+      var existingColumns = GetExistingColumns(connection, tableName);
+      if (existingColumns.Count == 0) {
+         var createColumns = string.Join(", ", columns.Select(column => $"{QuoteIdentifier(column)} TEXT"));
+         using var createCommand =
+            new SQLiteCommand($"CREATE TABLE IF NOT EXISTS {QuoteIdentifier(tableName)} ({createColumns})", connection);
+         createCommand.ExecuteNonQuery();
+         existingColumns = GetExistingColumns(connection, tableName);
+      }
+
+      foreach (var column in columns) {
+         if (existingColumns.Contains(column)) {
+            continue;
+         }
+
+         using var alterCommand =
+            new SQLiteCommand($"ALTER TABLE {QuoteIdentifier(tableName)} ADD COLUMN {QuoteIdentifier(column)} TEXT", connection);
+         alterCommand.ExecuteNonQuery();
+      }
+   }
+
+   private static HashSet<string> GetExistingColumns(SQLiteConnection connection, string tableName) {
+      var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+      using var command = new SQLiteCommand($"PRAGMA table_info({QuoteIdentifier(tableName)})", connection);
+      using var reader = command.ExecuteReader();
+      while (reader.Read()) {
+         columns.Add(reader["name"].ToString()!);
+      }
+
+      return columns;
+   }
+
+   private static void AddParameters(SQLiteCommand command, IReadOnlyDictionary<string, string> row) {
+      foreach (var pair in row) {
+         command.Parameters.AddWithValue(CreateParameterName(pair.Key), pair.Value);
+      }
+   }
+
+   private static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
+
+   private static string CreateParameterName(string columnName) => $"@p_{columnName.Replace("\"", string.Empty)}";
+
+   private static string CreateWhereParameterName(string columnName) => $"@w_{columnName.Replace("\"", string.Empty)}";
+}
@@ -0,0 +1,155 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Text.Json;
+
+public class PersistenceCallHandler {
+   private const string KeyValueTableName = "key_value_store";
+   private readonly PeristenceHelper _persistenceHelper;
+   private readonly PubSubSync _pub_sub_sync;
+
+   public PersistenceCallHandler(PeristenceHelper persistenceHelper, PubSubSync pub_sub_sync) {
+      _persistenceHelper = persistenceHelper;
+      _pub_sub_sync = pub_sub_sync;
+   }
+
+   public void Handle(HttpListenerRequest request, string payload) {
+      Console.WriteLine("----- Incoming REST call -----");
+      Console.WriteLine($"Timestamp: {DateTimeOffset.Now:O}");
+      Console.WriteLine($"Method: {request.HttpMethod}");
+      Console.WriteLine($"Url: {request.Url}");
+      Console.WriteLine($"RawUrl: {request.RawUrl}");
+      Console.WriteLine($"RemoteEndPoint: {request.RemoteEndPoint}");
+      Console.WriteLine($"UserAgent: {request.UserAgent}");
+      Console.WriteLine($"ContentType: {request.ContentType}");
+      Console.WriteLine($"ContentLength: {request.ContentLength64}");
+
+      if (request.QueryString.Count > 0) {
+         Console.WriteLine("Query:");
+         foreach (var key in request.QueryString.AllKeys) {
+            if (key is null) {
+               continue;
+            }
+
+            Console.WriteLine($"  {key} = {request.QueryString[key]}");
+         }
+      }
+
+      if (request.Headers.Count > 0) {
+         Console.WriteLine("Headers:");
+         foreach (var headerKey in request.Headers.AllKeys) {
+            if (headerKey is null) {
+               continue;
+            }
+
+            Console.WriteLine($"  {headerKey}: {request.Headers[headerKey]}");
+         }
+      }
+
+      Console.WriteLine("Body:");
+      Console.WriteLine(string.IsNullOrWhiteSpace(payload) ? "  <empty>" : payload);
+
+      var rpcRequest = ParseRpcRequest(payload);
+      HandleRpc(rpcRequest.FunctionName, rpcRequest.Arguments);
+      Console.WriteLine("------------------------------");
+   }
+
+   private static PersistenceRpcRequest ParseRpcRequest(string payload) {
+      if (string.IsNullOrWhiteSpace(payload)) {
+         throw new InvalidOperationException("RPC payload is empty.");
+      }
+
+      using var document = JsonDocument.Parse(payload);
+      var root = document.RootElement;
+
+      if (!root.TryGetProperty("function_name", out var functionNameElement) ||
+          functionNameElement.ValueKind != JsonValueKind.String) {
+         throw new InvalidOperationException("RPC payload must contain a string 'function_name'.");
+      }
+
+      if (!root.TryGetProperty("arguments", out var argumentsElement) ||
+          argumentsElement.ValueKind != JsonValueKind.Array) {
+         throw new InvalidOperationException("RPC payload must contain an 'arguments' array.");
+      }
+
+      var arguments = new List<string>();
+      foreach (var argument in argumentsElement.EnumerateArray()) {
+         if (argument.ValueKind != JsonValueKind.String) {
+            throw new InvalidOperationException("RPC arguments must all be strings.");
+         }
+
+         arguments.Add(argument.GetString()!);
+      }
+
+      return new PersistenceRpcRequest(functionNameElement.GetString()!, arguments.ToArray());
+   }
+
+   private void HandleRpc(string functionName, params string[] arguments) {
+      switch (functionName) {
+         case "keyvalue_insert":
+            RequireArgumentCount(functionName, arguments, 3);
+            HandleKeyValueInsert(arguments[0], arguments[1], arguments[2]);
+            break;
+         case "keyvalue_update":
+            RequireArgumentCount(functionName, arguments, 3);
+            HandleKeyValueUpdate(arguments[0], arguments[1], arguments[2]);
+            break;
+         case "keyvalue_delete":
+            RequireArgumentCount(functionName, arguments, 2);
+            HandleKeyValueDelete(arguments[0], arguments[1]);
+            break;
+         default:
+            throw new InvalidOperationException($"Unknown RPC function '{functionName}'.");
+      }
+   }
+
+   private static void RequireArgumentCount(string functionName, string[] arguments, int expectedCount) {
+      if (arguments.Length != expectedCount) {
+         throw new InvalidOperationException(
+            $"RPC function '{functionName}' expected {expectedCount} arguments but received {arguments.Length}.");
+      }
+   }
+
+   private void HandleKeyValueInsert(string itemNamespace, string key, string value) {
+      var row = new Dictionary<string, string> {
+         ["namespace"] = itemNamespace,
+         ["key"] = key,
+         ["value"] = value,
+      };
+      _persistenceHelper.InsertRow(KeyValueTableName, row);
+      _pub_sub_sync.publish_table(KeyValueTableName);
+      Console.WriteLine($"RPC: keyvalue_insert(namespace='{itemNamespace}', key='{key}', value='{value}')");
+   }
+
+   private void HandleKeyValueUpdate(string itemNamespace, string key, string value) {
+      var row = new Dictionary<string, string> {
+         ["namespace"] = itemNamespace,
+         ["key"] = key,
+         ["value"] = value,
+      };
+      _persistenceHelper.UpdateRow(KeyValueTableName, row);
+      _pub_sub_sync.publish_table(KeyValueTableName);
+      Console.WriteLine($"RPC: keyvalue_update(namespace='{itemNamespace}', key='{key}', value='{value}')");
+   }
+
+   private void HandleKeyValueDelete(string itemNamespace, string key) {
+      var row = new Dictionary<string, string> {
+         ["namespace"] = itemNamespace,
+         ["key"] = key,
+      };
+      _persistenceHelper.RemoveRow(KeyValueTableName, row);
+      _pub_sub_sync.publish_table(KeyValueTableName);
+      Console.WriteLine($"RPC: keyvalue_delete(namespace='{itemNamespace}', key='{key}')");
+   }
+}
+
+public record PersistenceRpcRequest {
+   public PersistenceRpcRequest(string functionName, string[] arguments) {
+      FunctionName = functionName;
+      Arguments = arguments;
+      Console.WriteLine($"PersistenceRpcRequest created for function '{FunctionName}' with {Arguments.Length} argument(s).");
+   }
+
+   public string FunctionName { get; init; }
+   public string[] Arguments { get; init; }
+}
@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,5 @@
+using Bts.PubSub;
+
+public class PersistenceServerConfig {
+   public PubSubConfig? pub_sub { get; set; }
+}
@@ -0,0 +1,113 @@
+using Bts.DataStreamCore;
+using BCG;
+
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+const int Port = 5079;
+var prefix = $"http://localhost:{Port}/";
+
+var listener = new HttpListener();
+listener.Prefixes.Add(prefix);
+var config = ConfigLoader.load();
+var persistenceHelper = new PeristenceHelper();
+var pub_sub_sync = new PubSubSync(persistenceHelper.database_path, config.pub_sub);
+var callHandler = new PersistenceCallHandler(persistenceHelper, pub_sub_sync);
+
+Console.WriteLine("PersistenceServer starting.");
+Console.WriteLine($"Listening on {prefix}");
+Console.WriteLine($"DataStreamCore loaded from: {typeof(IDataPacket).Assembly.FullName}");
+Console.WriteLine("Press Ctrl+C to stop.");
+
+var shutdown = new CancellationTokenSource();
+
+Console.CancelKeyPress += (_, eventArgs) => {
+   eventArgs.Cancel = true;
+   Console.WriteLine("Shutdown requested.");
+   shutdown.Cancel();
+   if (listener.IsListening) {
+      listener.Stop();
+   }
+};
+
+try {
+   pub_sub_sync.publish_all_tables();
+   listener.Start();
+   await RunAsync(listener, callHandler, shutdown.Token);
+   return 0;
+} catch (HttpListenerException ex) when (shutdown.IsCancellationRequested) {
+   Console.WriteLine($"Listener stopped: {ex.Message}");
+   return 0;
+} catch (Exception ex) {
+   Console.Error.WriteLine($"PersistenceServer crashed: {ex}");
+   return 1;
+} finally {
+   listener.Close();
+}
+
+static async Task RunAsync(
+   HttpListener listener,
+   PersistenceCallHandler callHandler,
+   CancellationToken cancellationToken) {
+   while (!cancellationToken.IsCancellationRequested) {
+      HttpListenerContext context;
+      try {
+         context = await listener.GetContextAsync();
+      } catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) {
+         break;
+      } catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) {
+         break;
+      }
+
+      _ = Task.Run(() => HandleRequestAsync(context, callHandler, cancellationToken), cancellationToken);
+   }
+}
+
+static async Task HandleRequestAsync(
+   HttpListenerContext context,
+   PersistenceCallHandler callHandler,
+   CancellationToken cancellationToken) {
+   var request = context.Request;
+   var response = context.Response;
+
+   try {
+      var body = await ReadBodyAsync(request, cancellationToken);
+      callHandler.Handle(request, body);
+
+      var responseText =
+         $$"""
+         {"status":"ok","method":"{{request.HttpMethod}}","path":"{{request.RawUrl ?? "/"}}"}
+         """;
+
+      var bytes = Encoding.UTF8.GetBytes(responseText);
+      response.StatusCode = (int)HttpStatusCode.OK;
+      response.ContentType = "application/json";
+      response.ContentEncoding = Encoding.UTF8;
+      response.ContentLength64 = bytes.LongLength;
+      await response.OutputStream.WriteAsync(bytes, cancellationToken);
+   } catch (Exception ex) {
+      Console.Error.WriteLine($"Request handling failed: {ex}");
+      response.StatusCode = (int)HttpStatusCode.InternalServerError;
+   } finally {
+      response.OutputStream.Close();
+      response.Close();
+   }
+}
+
+static async Task<string> ReadBodyAsync(HttpListenerRequest request, CancellationToken cancellationToken) {
+   if (!request.HasEntityBody) {
+      return string.Empty;
+   }
+
+   using var reader = new StreamReader(
+      request.InputStream,
+      request.ContentEncoding ?? Encoding.UTF8,
+      detectEncodingFromByteOrderMarks: true,
+      leaveOpen: true);
+
+   return await reader.ReadToEndAsync(cancellationToken);
+}
@@ -0,0 +1,116 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SQLite;
+
+using BCG;
+using Bts.DataStreamCore;
+using Bts.PubSub;
+using Bts.PubSub.Routing;
+using Microsoft.Extensions.DependencyInjection;
+
+using bts;
+
+public class PubSubSync {
+   private readonly string _connectionString;
+   private readonly Dictionary<string, IPubTableBroker> _brokers = new(StringComparer.OrdinalIgnoreCase);
+   private readonly object _sync_root = new();
+   private readonly IServiceProvider _serviceProvider;
+   private readonly PubSubRouterConfig _router_config;
+   private readonly PubSubTagHandlerFactoryConfig _tag_handler_factory_config;
+
+   public PubSubSync(string databasePath, PubSubConfig? pubSubConfig = null) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(databasePath);
+      _connectionString = $"Data Source={databasePath};Version=3";
+      var resolved_pub_sub_config = get_or_initialize_pub_sub_config(pubSubConfig);
+      _router_config = resolved_pub_sub_config.Routing ?? throw new InvalidOperationException("PubSub routing config is required.");
+      _tag_handler_factory_config = resolved_pub_sub_config.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default();
+      _serviceProvider = new ServiceCollection().BuildServiceProvider();
+   }
+
+   public void publish_all_tables() {
+      lock (_sync_root) {
+         foreach (var table_name in get_table_names()) {
+            publish_table_core(table_name);
+         }
+      }
+   }
+
+   public void publish_table(string table_name) {
+      ArgumentException.ThrowIfNullOrWhiteSpace(table_name);
+
+      lock (_sync_root) {
+         publish_table_core(table_name);
+      }
+   }
+
+   private void publish_table_core(string table_name) {
+      var rows = load_table(table_name);
+      var broker = get_or_create_broker(table_name);
+      broker.create_table(rows);
+      Console.WriteLine($"PubSubSync published table '{table_name}' with {rows.Count} row(s).");
+   }
+
+   private IPubTableBroker get_or_create_broker(string table_name) {
+      if (_brokers.TryGetValue(table_name, out var broker)) {
+         return broker;
+      }
+
+      var publisher_core = new PublisherCore(
+         new PubSubRouter(_router_config),
+         new PubsubTagHandlerFactory(_serviceProvider, _tag_handler_factory_config));
+      broker = new PubTableBroker(publisher_core);
+      _brokers[table_name] = broker;
+      return broker;
+   }
+
+   private string[] get_table_names() {
+      using var connection = open_connection();
+      using var command = new SQLiteCommand(
+         "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
+         connection);
+      using var reader = command.ExecuteReader();
+
+      var table_names = new List<string>();
+      while (reader.Read()) {
+         table_names.Add(reader.GetString(0));
+      }
+
+      return table_names.ToArray();
+   }
+
+   private IBlueRowCollection load_table(string table_name) {
+      using var connection = open_connection();
+      using var adapter = new SQLiteDataAdapter($"SELECT * FROM {quote_identifier(table_name)}", connection);
+      var table = new DataTable(table_name);
+      adapter.Fill(table);
+      table.TableName = table_name;
+      return BlueDataUtility.toIBlueRowCollection(table);
+   }
+
+   private SQLiteConnection open_connection() {
+      var connection = new SQLiteConnection(_connectionString);
+      connection.Open();
+      return connection;
+   }
+
+   private static PubSubConfig get_or_initialize_pub_sub_config(PubSubConfig? configured_pub_sub) {
+      try {
+         PubSubConfig.SetGlobalConfig(PubSubConfig.Default());
+         var existing = PubSubConfig.Instance;
+         if (configured_pub_sub is not null) {
+            existing.MergeWith(configured_pub_sub);
+         }
+         return existing;
+      } catch (InvalidOperationException) {
+         var config = PubSubConfig.Default();
+         if (configured_pub_sub is not null) {
+            config.MergeWith(configured_pub_sub);
+         }
+         PubSubConfig.SetGlobalConfig(config);
+         return config;
+      }
+   }
+
+   private static string quote_identifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
+}
@@ -0,0 +1,20 @@
+{
+  "PubSub": {
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    }
+  }
+}
@@ -0,0 +1,43 @@
+import json
+import sys
+import urllib.error
+import urllib.request
+
+
+SERVER_URL = "http://localhost:5079/save?source=python-client"
+
+
+def main() -> int:
+   payload = {
+      "function_name": "keyvalue_insert",
+      "arguments": [
+         "example-namespace",
+         "example-key",
+         "hello from python",
+      ],
+   }
+
+   data = json.dumps(payload).encode("utf-8")
+   request = urllib.request.Request(
+      SERVER_URL,
+      data=data,
+      headers={
+         "Content-Type": "application/json",
+         "X-Client": "PersistenceServerPythonClient",
+      },
+      method="POST",
+   )
+
+   try:
+      with urllib.request.urlopen(request) as response:
+         body = response.read().decode("utf-8")
+         print(f"status: {response.status}")
+         print(f"body: {body}")
+         return 0
+   except urllib.error.URLError as error:
+      print(f"request failed: {error}", file=sys.stderr)
+      return 1
+
+
+if __name__ == "__main__":
+   raise SystemExit(main())
@@ -1,44 +1,44 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Runtime.InteropServices;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Runtime.InteropServices;

-namespace BCG
-{
-   public class ClientToServerConnectionWrapper
-   {
-      [DllImport("BlutoInterop.dll")]
-      public unsafe static extern void init();
+// namespace BCG
+// {
+//    public class ClientToServerConnectionWrapper
+//    {
+//       [DllImport("BlutoInterop.dll")]
+//       public unsafe static extern void init();

-      [DllImport("BlutoInterop.dll")]
-      public unsafe static extern void initServerConnection(byte[] ip, int port);
+//       [DllImport("BlutoInterop.dll")]
+//       public unsafe static extern void initServerConnection(byte[] ip, int port);

-      [DllImport("BlutoInterop.dll")]
-      public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
+//       [DllImport("BlutoInterop.dll")]
+//       public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);

-      [DllImport("BlutoInterop.dll")]
-      public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
+//       [DllImport("BlutoInterop.dll")]
+//       public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);

-      public static byte[] stringToBytes(string s)
-      {
-         byte[] rv = new byte[s.Length + 1];
-         for (int i = 0; i < s.Length; i++)
-            rv[i] = Convert.ToByte(s[i]);
-         rv[s.Length] = Convert.ToByte ('\0');
-         return rv;
-      }
+//       public static byte[] stringToBytes(string s)
+//       {
+//          byte[] rv = new byte[s.Length + 1];
+//          for (int i = 0; i < s.Length; i++)
+//             rv[i] = Convert.ToByte(s[i]);
+//          rv[s.Length] = Convert.ToByte ('\0');
+//          return rv;
+//       }

-      public static string intPtrToString(IntPtr iptr)
-      {
-         return Marshal.PtrToStringAnsi(iptr);
-      }
+//       public static string intPtrToString(IntPtr iptr)
+//       {
+//          return Marshal.PtrToStringAnsi(iptr);
+//       }

-      public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
-      {
-         byte[] data = new byte[sz];
-         Marshal.Copy(iptr, data, 0, data.Length);
-         return data;
-      }
+//       public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
+//       {
+//          byte[] data = new byte[sz];
+//          Marshal.Copy(iptr, data, 0, data.Length);
+//          return data;
+//       }

-   }
-}
+//    }
+// }
@@ -39,8 +39,6 @@ public static class PubSubMessageUtils {
   }
 }

-// Subscribe //////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
 public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
   public string? connection;
   public string subscribing_app = ClientInfo.thisClient.app;
@@ -86,8 +84,6 @@ public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
      $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {string.Concat(tags.empty_if_null())}";
 }

-// Unsubscribe ////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
 public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
   public bool Equals(UnsubscribeMsg? other) =>
      other is not null && base.Equals(other);
@@ -99,8 +95,6 @@ public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
      base.GetHashCode();
 }

-// Trasnaction /////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////
 public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
   uint n_msgs;

@@ -127,15 +121,12 @@ public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
      n_msgs.GetHashCode();
 }

-// Publish ////////////////////////////////////////////////////////////////
-/// ///////////////////////////////////////////////////////////////////////
 public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
   public string? connection;
   public string publishing_app = ClientInfo.thisClient.app;
   public string? tag;
   public string? tag_type;
   public bool is_compressed = false;
-   // this is the size that of the msg, regarless of it being serialized or not
   public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
   public byte[]? data_to_publish;

@@ -144,8 +135,6 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
      Log.assert(!string.IsNullOrEmpty(tag_type));
      Log.assert(!string.IsNullOrEmpty(tag));
      Log.assert(!string.IsNullOrEmpty(connection));
-      // The 'ThrowIfNullOrEmpty' calls are redundant (given the assert statements above), but
-      // necessary for the compiler to recognize the strings won't be null when this method exits.
      ArgumentException.ThrowIfNullOrEmpty(tag_type);
      ArgumentException.ThrowIfNullOrEmpty(tag);
      ArgumentException.ThrowIfNullOrEmpty(connection);
@@ -181,7 +170,7 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
   }

   public bool Equals(PublishMsg? other) =>
-      other is not null 
+      other is not null
         && connection == other.connection
         && publishing_app == other.publishing_app
         && tag == other.tag
@@ -220,7 +209,6 @@ public enum PubSubMessageType {

 public class PubSubClientMessage : IPubSubMessage {
   public PubSubMessageType msg_type;
-   // only one of these wil be non null
   public SubscribeMsg? sub_msg;
   public UnsubscribeMsg? unsub_msg;
   public TransactionMsg? transaction_msg;
@@ -265,7 +253,7 @@ public class PubSubClientMessage : IPubSubMessage {
            ValidateMsg(pub_msg).serialize_into_str(str);
            break;

-         T ValidateMsg<T>([NotNull]T? msg, [CallerArgumentExpression(nameof(msg))]string name = "") {
+         T ValidateMsg<T>([NotNull] T? msg, [CallerArgumentExpression(nameof(msg))] string name = "") {
            if (msg is null) {
               throw new InvalidOperationException($"msg type is {msg_type} but {name} is null");
            }
@@ -383,7 +371,7 @@ public class PubSubServerMessage : IPubSubMessage, IEquatable<PubSubServerMessag

   [MemberNotNull(nameof(tag), nameof(tag_type))]
   public void construct_deserialize(byte[] msg) =>
-      construct_deserialize(new MemoryStream(msg) { Position = 0});
+      construct_deserialize(new MemoryStream(msg) { Position = 0 });

   public bool Equals(PubSubServerMessage? other) =>
      other is not null
@@ -17,11 +17,6 @@ public class PubSubTagHandlerFactoryConfig {
   public static PubSubTagHandlerFactoryConfig Default() {
      return new() {
         Handlers = new() {
-            new() {
-               TagType = string.Empty,
-               Assembly = "DataStreamCore",
-               Class = "bts.PubsubServerStreamTagHandler",
-            },
            new() {
               TagType = DSTagManager.DataStreamTagType,
               Assembly = "DataStreamCore",
@@ -36,7 +31,12 @@ public class PubSubTagHandlerFactoryConfig {
               TagType = "System_Stats",
               Assembly = "PubSub",
               Class = "BCG.SystemStatsPublisher",
-            }
+            },
+            new() {
+               TagType = ".*",
+               Assembly = "DataStreamCore",
+               Class = "bts.PubsubServerGenericDataHandler",
+            },
         }
      };
   }
@@ -48,16 +48,16 @@ public class PubSubConfig {
         HeartbeatInterval = 15000,
         MissingHeartbeatsBeforeGc = 50,
         DisposedConnectionCheckPeriod = 1000,
-         ServerPort = 12345,
+         ServerPort = 22019,
         TagTypeToAssemblyMap = new(),
         TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(),
         SimpleEchoServer = false,
         Routing = new() {
            PublisherRoutes = new() {
-               new("bts-cloud://127.0.0.1:12345", ".*")
+               new("bts-cloud://127.0.0.1:22019", ".*")
            },
            SubscriberRoutes = new() {
-               new("bts-cloud://127.0.0.1:12345", ".*")
+               new("bts-cloud://127.0.0.1:22019", ".*")
            }
         }
      };
@@ -12,6 +12,7 @@ using System.Net.Sockets;

 // opens a TCP port and starts listening to connections
 public class AsyncTCPConnectionAcceptor_APM {
+   // This acceptor exists as an alternate implementation and is not the one currently used by the pubsub server.
   private readonly int _port;
   Socket? _accept_socket;
   readonly ITCPConnectionHandler _connection_handler;
@@ -13,6 +13,7 @@ namespace bts {

   // opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
   // it uses TcpListener, instead of raw sockets.
+   // This is the acceptor currently used by the pubsub server via ManagedTCPMessageHub.
   public class AsyncTCPConnectionAcceptor_AsyncWrapper {
      private readonly int _accept_port;
      // private Socket _accept_socket;
@@ -10,6 +10,7 @@ namespace bts {
    // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
    // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
    // statefullness and hence non static
+    // This connector exists as an alternate implementation and is not the one currently used by pubsub clients.
    public static class AsyncTCPConnector_APM {

        internal class SocketConnectionArgs {
@@ -13,17 +13,18 @@ using System.Diagnostics.CodeAnalysis;


 namespace bts {
-   // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
-   // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
+   // Unlike AsyncTCPConnectionAcceptor_AsyncWrappe this class is static since the requests to connect are stand alone requests
+   // whereas AsyncTCPConnectionAcceptor_AsyncWrappe is always listening for incoming connections which lends itself to more
   // statefullness and hence non static
+   // This is the connector currently used by pubsub clients via PubSubServerManagedTCPConnection.
   public static class AsyncTCPConnector_AsyncWrapper {
      public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
         bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}");
         //Task.Run(() => _connect_to(server_address, handler));
         ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(
-            Connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
+            _connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));

-         bool Connect() {
+         bool _connect() {
            try {
               return _connect_to(server_address, handler);
            } catch (Exception ex) {
@@ -49,7 +50,7 @@ namespace bts {
         handler.onConnection(conn);
         return true;

-         bool CreateTcpClient([NotNullWhen(true)]out TcpClient? tcpClient) {
+         bool CreateTcpClient([NotNullWhen(true)] out TcpClient? tcpClient) {
            try {
               tcpClient = new(server_rci.ip, server_rci.port);
               return true;
@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Text;

 namespace BCG
@@ -24,8 +24,8 @@ namespace BCG
      {
         get { return myApp; }
         set { myApp = value; }
-
      }
+
      private string myKeyQualifier;
      public string keyQualifier
      {
@@ -38,8 +38,8 @@ namespace BCG
      {
         get { return myStartTime; }
         set { myStartTime = value; }
-
      }
+
      string? myProcessString;
      public string processString
      {
@@ -49,10 +49,8 @@ namespace BCG
               myProcessString = getProcessString(";");
            return myProcessString;
         }
-
      }

-      
      public ClientInfo(string theMachineName, int thePid, string theProcessName, string theKeyQualifier, DateTime theStartTime)
      {
         myMachineName = theMachineName;
@@ -68,10 +66,8 @@ namespace BCG
         myMachineName = Environment.MachineName;
         myPid = System.Diagnostics.Process.GetCurrentProcess().Id;
         myApp = System.Diagnostics.Process.GetCurrentProcess().ProcessName.ToUpper();
-         string[] args = System.Environment.GetCommandLineArgs();
+         string[] args = Environment.GetCommandLineArgs();

-         //the key qualifier is the last argument given. If they didn't give any arguments
-         //but the program itself, then the KQ is empty. 
         if (args.Length > 1)
            myKeyQualifier = args[args.Length - 1];
         else
@@ -86,13 +82,13 @@ namespace BCG

      public static ClientInfo thisClient
      {
-         get 
+         get
         {
            if (myLocalClient == null)
            {
-               lock(myLock)
+               lock (myLock)
               {
-                  if(myLocalClient == null)
+                  if (myLocalClient == null)
                     myLocalClient = new ClientInfo();
               }
            }
@@ -102,7 +98,6 @@ namespace BCG

      public static ClientInfo fromProcessString(string procString)
      {
-
         ClientInfo outputGuy = new ClientInfo();

         string[] info = procString.Split(';');
@@ -112,9 +107,7 @@ namespace BCG
         outputGuy.myKeyQualifier = info[2];
         outputGuy.myPid = Int32.Parse(info[3]);

-
         return outputGuy;
-
      }

      private string getProcessString(string seperator)
@@ -1,273 +0,0 @@
-<?xml version="1.0" ?>
-<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <PropertyGroup>
-    <ProjectType>Local</ProjectType>
-    <ProductVersion>9.0.30729</ProductVersion>
-    <SchemaVersion>2.0</SchemaVersion>
-    <ProjectGuid>{41DBB1A3-1A4E-457D-BB2C-BFDE486E328B}</ProjectGuid>
-    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
-    <Platform Condition=" '$(Platform)' == '' ">x64</Platform>
-    <ApplicationIcon>
-    </ApplicationIcon>
-    <AssemblyKeyContainerName>
-    </AssemblyKeyContainerName>
-    <AssemblyName>CompressionUtilities</AssemblyName>
-    <AssemblyOriginatorKeyFile>..\..\StrongNameKeys\Builder.snk</AssemblyOriginatorKeyFile>
-    <DefaultClientScript>JScript</DefaultClientScript>
-    <DefaultHTMLPageLayout>Grid</DefaultHTMLPageLayout>
-    <DefaultTargetSchema>IE50</DefaultTargetSchema>
-    <DelaySign>false</DelaySign>
-    <OutputType>Library</OutputType>
-    <RootNamespace>CompressionUtilities</RootNamespace>
-    <RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
-    <StartupObject>
-    </StartupObject>
-    <SignAssembly>false</SignAssembly>
-    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
-    <ApplicationRevision>0</ApplicationRevision>
-    <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
-    <BaseAddress>285212672</BaseAddress>
-    <CheckForOverflowUnderflow>false</CheckForOverflowUnderflow>
-    <ConfigurationOverrideFile>
-    </ConfigurationOverrideFile>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <DocumentationFile>
-    </DocumentationFile>
-    <DebugSymbols>true</DebugSymbols>
-    <FileAlignment>4096</FileAlignment>
-    <NoStdLib>false</NoStdLib>
-    <NoWarn>
-    </NoWarn>
-    <Optimize>false</Optimize>
-    <RegisterForComInterop>false</RegisterForComInterop>
-    <RemoveIntegerChecks>false</RemoveIntegerChecks>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <WarningLevel>4</WarningLevel>
-    <DebugType>full</DebugType>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
-    <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
-    <BaseAddress>285212672</BaseAddress>
-    <CheckForOverflowUnderflow>false</CheckForOverflowUnderflow>
-    <ConfigurationOverrideFile>
-    </ConfigurationOverrideFile>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <DocumentationFile>
-    </DocumentationFile>
-    <DebugSymbols>true</DebugSymbols>
-    <FileAlignment>4096</FileAlignment>
-    <NoStdLib>false</NoStdLib>
-    <NoWarn>
-    </NoWarn>
-    <Optimize>false</Optimize>
-    <RegisterForComInterop>false</RegisterForComInterop>
-    <RemoveIntegerChecks>false</RemoveIntegerChecks>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <WarningLevel>4</WarningLevel>
-    <DebugType>full</DebugType>
-    <ErrorReport>prompt</ErrorReport>
-    <DefineDebug>true</DefineDebug>
-    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Test|AnyCPU' ">
-    <DebugSymbols>true</DebugSymbols>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <BaseAddress>285212672</BaseAddress>
-    <DebugType>full</DebugType>
-    <PlatformTarget>x64</PlatformTarget>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
-    <DebugSymbols>true</DebugSymbols>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <BaseAddress>285212672</BaseAddress>
-    <FileAlignment>4096</FileAlignment>
-    <DebugType>full</DebugType>
-    <PlatformTarget>x64</PlatformTarget>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
-    <DebugSymbols>true</DebugSymbols>
-    <DefineDebug>true</DefineDebug>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <BaseAddress>285212672</BaseAddress>
-    <FileAlignment>4096</FileAlignment>
-    <DebugType>full</DebugType>
-    <PlatformTarget>x64</PlatformTarget>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Test|x64' ">
-    <DebugSymbols>true</DebugSymbols>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <BaseAddress>285212672</BaseAddress>
-    <DebugType>full</DebugType>
-    <PlatformTarget>x64</PlatformTarget>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
-  </PropertyGroup>
-  <ItemGroup>
-    <Reference Include="System">
-      <Name>System</Name>
-    </Reference>
-    <Reference Include="System.Core">
-      <RequiredTargetFramework>4.0</RequiredTargetFramework>
-    </Reference>
-    <Reference Include="System.Data">
-      <Name>System.Data</Name>
-    </Reference>
-    <Reference Include="System.Xml">
-      <Name>System.XML</Name>
-    </Reference>
-  </ItemGroup>
-  <ItemGroup>
-    <Compile Include="AssemblyInfo.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="BlueCompressUtilities\IBlueCompressor.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="BlueCompressUtilities\SharpZipCompressor.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\BZip2\BZip2.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\BZip2\BZip2Constants.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\BZip2\BZip2InputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\BZip2\BZip2OutputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Checksums\Adler32.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Checksums\Crc32.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Checksums\IChecksum.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Checksums\StrangeCrc.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\GZip\GZipConstants.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\GZip\GZipInputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\GZip\GZipOutputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\InvalidHeaderException.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\TarArchive.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\TarBuffer.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\TarEntry.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\TarHeader.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\TarInputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Tar\TarOutputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\ZipException.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\Deflater.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\DeflaterConstants.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\DeflaterEngine.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\DeflaterHuffman.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\DeflaterPending.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\Inflater.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\InflaterDynHeader.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\InflaterHuffmanTree.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\PendingBuffer.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\Streams\DeflaterOutputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\Streams\InflaterInputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\Streams\OutputWindow.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\Compression\Streams\StreamManipulator.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\ZipConstants.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\ZipEntry.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\ZipFile.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\ZipInputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-    <Compile Include="SharpZipLib\Zip\ZipOutputStream.cs">
-      <SubType>Code</SubType>
-    </Compile>
-  </ItemGroup>
-  <ItemGroup>
-    <BootstrapperPackage Include="Microsoft.Net.Client.3.5">
-      <Visible>False</Visible>
-      <ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
-    </BootstrapperPackage>
-    <BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
-      <Visible>False</Visible>
-      <ProductName>.NET Framework 3.5 SP1</ProductName>
-    </BootstrapperPackage>
-    <BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
-      <Visible>False</Visible>
-      <ProductName>Windows Installer 3.1</ProductName>
-    </BootstrapperPackage>
-  </ItemGroup>
-  <Import Project="..\..\build\blue2.cs.targets"/>
-  <PropertyGroup>
-    <PreBuildEvent>
-    </PreBuildEvent>
-    <PostBuildEvent>
-    </PostBuildEvent>
-  </PropertyGroup>
-</Project>
@@ -3,7 +3,7 @@ using System;
 namespace BCG.Utilities.General
 {
   public interface IWorker
-	{
+   {
      void doWork();
-	}
+   }
 }
@@ -79,5 +79,4 @@ namespace mbts
         return ToString().GetHashCode();
      }
   }
-
 }
@@ -9,7 +9,7 @@ namespace BCG {
   /// <summary>
   /// Collection of utility methods used throughout
   /// </summary>
-	public static class StreamUtilityMethods {
+   public static class StreamUtilityMethods {

      // extension method to write a string C++ style (len prefixed)
      public static void write_string_cpp(this BinaryWriter bw, string? str) {
@@ -98,7 +98,7 @@ namespace BCG {
         byte[] secondStreamBytes = new byte[stream.Length - splitLocation];
         for (int i = 0; i < stream.Length; ++i) {
            //for some resaon, the 'getBytesFromStream' function prepends the length of the byte array to the bytes. This means
-            //we have to ignore those bytes here. 
+            //we have to ignore those bytes here.
            if (i < splitLocation)
               firstStreamBytes[i] = streamBytes[i + 4];
            else
@@ -107,8 +107,6 @@ namespace BCG {
         }

         return new Stream[] { new MemoryStream(firstStreamBytes), new MemoryStream(secondStreamBytes) };
-
-
      }
   }
 }
@@ -4,17 +4,20 @@
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
    <TargetFramework>net8.0</TargetFramework>
    <Nullable>enable</Nullable>
+    <PackageId>Bts.PubSub</PackageId>
  </PropertyGroup>

  <Choose>
    <When Condition="'$(UseProjectRefs)' == 'true'">
      <ItemGroup>
+        <ProjectReference Include="..\..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj" />
        <ProjectReference Include="..\..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj" />
-        <ProjectReference Include="..\Bts.Net\Bts.Net.csproj" />
+        <ProjectReference Include="..\..\cs\Bts.Net\Bts.Net.csproj" />
      </ItemGroup>
    </When>
    <Otherwise>
      <ItemGroup>
+        <PackageReference Include="Bts.IO.Compression" Version="$(SparkVersion)" />
        <PackageReference Include="Bts.Spark.Core" Version="$(SparkVersion)" />
        <PackageReference Include="Bts.Net" Version="$(SparkVersion)" />
      </ItemGroup>
@@ -4,7 +4,8 @@ using System;
 using System.Collections.Generic;

 namespace bts {
-   // This caches all the published data. This class is created and held by
+   // This caches all the published data, on the pubsliher (client) side. 
+   // This class is created and held by
   // PublisherCore. When data is publsihed , PublisherCore passes it to this 
   // class via the publish call. On a reconnection to the PubSubServer, 
   // PublisherCore calls get_content to get the data to re-publish, to 
@@ -43,17 +43,22 @@ namespace BCG {

      #region constructors

-      public PublisherCore(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null, IServiceProvider? serviceProvider = null) {
-         router ??= new PubSubRouter();
-         serviceProvider ??= PubSubServices.ServiceProvider;
-         tagHandlerFactory ??= new PubsubTagHandlerFactory(serviceProvider, PubSubTagHandlerFactoryConfig.Default());
+      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tag_handler_factory) {
         _router = router;
-         _state = new(tagHandlerFactory);
+         _state = new(tag_handler_factory);
         Thread pump_thread = new Thread(new ThreadStart(_message_pump));
         pump_thread.IsBackground = true;
         pump_thread.Start();
      }

+      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tag_handler_factory = null, IServiceProvider? service_provider = null) {
+         router ??= new PubSubRouter();
+         tag_handler_factory ??= new PubsubTagHandlerFactory(
+            service_provider ?? PubSubServices.ServiceProvider,
+            PubSubConfig.Instance.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default());
+         return new(router, tag_handler_factory);
+      }
+
      public void Dispose() { }
      #endregion

@@ -114,7 +119,7 @@ namespace BCG {
      #endregion

      // static void setGlobalInstance() {
-         
+
      // }

      #region Message Pump Thread - this is where messages are dequeued and the action performed
@@ -2,6 +2,6 @@ namespace Bts.PubSub.Server;

 using bts;

-interface IInternalPublisher {
+public interface IInternalPublisher {
   void Publish(PublishMsg message);
 }
@@ -10,7 +10,7 @@ using System.Collections.Generic;
 /// messages to be published. It is also passed to the <see cref="BCG.PubSubMessageReactor"/>, so it
 /// can dequeue these messages and send them over the wire.
 /// </summary>
-interface IInternalPublisherQueue {
+public interface IInternalPublisherQueue {
   void Enqueue(PublishMsg message);
   List<PublishMsg> DequeueMultiple(TimeSpan ts, int maxElements);
 }
@@ -0,0 +1,9 @@
+namespace BlueOrb.PubSub.Server;
+
+using BCG.Utilities.General;
+
+using System;
+
+public interface IPubSubInternalPublisher : IWorker {
+   event Action<string, string, byte[]>? onNewData;
+}
@@ -10,6 +10,16 @@ public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEven
 public delegate void PubSubStateChangeDelegate(string[] tags);
 public delegate void PubSubStatsDelegate(string tag, int size);

+public class PubSubServerTagStats {
+   public int n_updates { get; set; }
+   public long total_bytes { get; set; }
+   public int min_size { get; set; }
+   public int max_size { get; set; }
+   public int last_update_siz { get; set; }
+   public int average_size { get; set; }
+   public long last_update_time { get; set; }
+}
+
 public interface IPubSubState {
   event PubSubStateChangeDelegate? onTagMapChanged;
   event PubSubStatsDelegate? onPublishStats;
@@ -18,6 +28,7 @@ public interface IPubSubState {

   byte[] get_content(string tag);
   string getTagType(string tag);
+   PubSubServerTagStats get_stats(string tag);
   void get_all_connections(
      string tag,
      out Dictionary<string, string> subscribers,
@@ -7,7 +7,7 @@ using bts;
 using System;
 using System.Collections.Generic;

-class InternalPublisherQueue : IInternalPublisherQueue {
+public class InternalPublisherQueue : IInternalPublisherQueue {
   readonly SyncQueue<PublishMsg> queue = new();
   readonly object lockObj = new();

@@ -1,72 +1,76 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using System.IO;
-using bts;
-using Bts.PubSub;
-using Bts.PubSub.Server;
-using Spark.Core;
+// using System;
+// using BCG.Utilities.General;
+// using System.IO;
+// using bts;
+// using Bts.PubSub;
+// using Bts.PubSub.Server;
+// using Spark.Core;

-namespace BCG {
-   class Pubsubheartbeat {
-      public Int64 hb_time_epoch;
-      public byte[] serialize() {
-         MemoryStream str = new MemoryStream();
-         BinaryWriter bw = new BinaryWriter(str);
-         bw.Write(hb_time_epoch);
-         str.Position = 0;
-         return str.ToArray();
-      }
+// namespace BCG {
+//    class Pubsubheartbeat {
+//       public Int64 hb_time_epoch;
+//       public byte[] serialize() {
+//          MemoryStream str = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(str);
+//          bw.Write(hb_time_epoch);
+//          str.Position = 0;
+//          return str.ToArray();
+//       }

-      public Pubsubheartbeat() {
-         hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-      }
+//       public Pubsubheartbeat() {
+//          hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+//       }

-      public static Pubsubheartbeat deserialize(byte[] data) {
-         long hb_time = BitConverter.ToInt64(data);
-         return new Pubsubheartbeat() {
-            hb_time_epoch = hb_time,
-         };
-      }
-   }
-   /// <summary>
-   /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
-   /// A subscriber can subscribe to it, and find out of a PubSub is alive or 
-   /// not. Infact more so, if the subsriptions are working or not.
-   /// </summary>
-   internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
-      public static string tag_type = "System_Heartbeat";
-      public static string tag_name = "System_Heartbeat";
-      readonly IInternalPublisher internalPublisher;
-      readonly IWorker workerImpl;
+//       public static Pubsubheartbeat deserialize(byte[] data) {
+//          long hb_time = BitConverter.ToInt64(data);
+//          return new Pubsubheartbeat() {
+//             hb_time_epoch = hb_time,
+//          };
+//       }
+//    }
+//    /// <summary>
+//    /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
+//    /// A subscriber can subscribe to it, and find out if a PubSubServer is alive 
+//    /// or not. Infact more so, if the subsriptions are working or not.
+//    /// </summary>
+//    internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
+//       public static string tag_type = "System_Heartbeat";
+//       public static string tag_name = "System_Heartbeat";

-      public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
-         this.internalPublisher = internalPublisher;
-         workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
-      }
+//       readonly IInternalPublisher internalPublisher;
+//       readonly IWorker workerImpl;

-      TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
+//       public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
+//          this.internalPublisher = internalPublisher;
+//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
+//       }

-      #region IWorker Members
+//       TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);

-      public void doWork() =>
-         workerImpl.doWork();
+//       #region IWorker Members

-      void doWorkImpl() {
-         Pubsubheartbeat hb = new Pubsubheartbeat();
-         PublishMsg msg = new PublishMsg() {
-            connection = "Internal",
-            publishing_app = ClientInfo.thisClient.app,
-            tag = "pss_hb",
-            tag_type = "Internal",
-            is_compressed = false,
-            publish_time_epoch_ms = hb.hb_time_epoch,
-            data_to_publish = hb.serialize()
-         };
-         internalPublisher.Publish(msg);
-      }
+//       public void doWork() =>
+//          workerImpl.doWork();

-      #endregion
-   }
-}
+//       void doWorkImpl() {
+//          // why not publish a table
+//          Pubsubheartbeat hb = new Pubsubheartbeat();
+//          PublishMsg msg = new PublishMsg() {
+//             connection = "Internal",
+//             publishing_app = ClientInfo.thisClient.app,
+//             tag = "pss_hb",
+//             tag_type = "Internal",
+//             is_compressed = false,
+//             publish_time_epoch_ms = hb.hb_time_epoch,
+//             data_to_publish = hb.serialize()
+//          };
+//          internalPublisher.Publish(msg);
+//       }
+
+//       #endregion
+
+//       private void _publish_hb() {
+
+//       }
+//    }
+// }
@@ -1,97 +1,97 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
-using BCG.Utilities.General;
-using bts;
-using Bts.PubSub;
-using Bts.PubSub.Server;
-using Spark.Core;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+// using BCG.Utilities.General;
+// using bts;
+// using Bts.PubSub;
+// using Bts.PubSub.Server;
+// using Spark.Core;

-namespace BCG {
-   /// <summary>
-   /// Very similiar to SystemSubscriptionMapPublisher, except that it keeps updating the state as _handle_publish
-   /// is called (in_tag_stats_map). in do_work it goes through the ones who were recently updated and call an internal_publish
-   /// Simialiar to SystemSubscriptionMapPublisher, it is one half of an IPubSubServerTagHandler, ie does not react to update,
-   /// but returns the state in get_content (though it could have perhaps used update instead ofth spcialized event, onPublishStats) to 
-   /// keep track of publish sizes.
-   /// </summary>
-   class SystemStatsPublisher : VirtualSingleton<SystemStatsPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
-      public static string tag_type = "System_Stats";
-      public static string tag_name = "System_AllStatistics";
-      // these tags have been touched, need to publish in the next doWork
-      private HashSet<string> _dirty_tags = new HashSet<string>();
-      private Dictionary<string, TagStatsInfo> _tag_stats_map = new Dictionary<string, TagStatsInfo>();
-      readonly IInternalPublisher internalPublisher;
-      readonly IWorker workerImpl;
+// namespace BCG {
+//    /// <summary>
+//    /// Very similiar to SystemSubscriptionMapPublisher, except that it keeps updating the state as _handle_publish
+//    /// is called (in_tag_stats_map). in do_work it goes through the ones who were recently updated and call an internal_publish
+//    /// Simialiar to SystemSubscriptionMapPublisher, it is one half of an IPubSubServerTagHandler, ie does not react to update,
+//    /// but returns the state in get_content (though it could have perhaps used update instead ofth spcialized event, onPublishStats) to 
+//    /// keep track of publish sizes.
+//    /// </summary>
+//    class SystemStatsPublisher : VirtualSingleton<SystemStatsPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
+//       public static string tag_type = "System_Stats";
+//       public static string tag_name = "System_AllStatistics";
+//       // these tags have been touched, need to publish in the next doWork
+//       private HashSet<string> _dirty_tags = new HashSet<string>();
+//       private Dictionary<string, TagStatsInfo> _tag_stats_map = new Dictionary<string, TagStatsInfo>();
+//       readonly IInternalPublisher internalPublisher;
+//       readonly IWorker workerImpl;

-      public SystemStatsPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
-         bts.Log.Gen.info("SystemStatsPublisher constructed");
-         workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
-         this.internalPublisher = internalPublisher;
-         state.onPublishStats += _handle_publish;
-      }
+//       public SystemStatsPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
+//          bts.Log.Gen.info("SystemStatsPublisher constructed");
+//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
+//          this.internalPublisher = internalPublisher;
+//          state.onPublishStats += _handle_publish;
+//       }

-      public string TagType => tag_type;
-      TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250);
+//       public string TagType => tag_type;
+//       TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250);

-      void _handle_publish(string tag, int size) {
-         bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}");
-         if (tag == tag_name)
-            return;
-         if (!_tag_stats_map.TryGetValue(tag, out var tag_stats)) {
-            tag_stats = new TagStatsInfo(tag);
-            _tag_stats_map.Add(tag, tag_stats);
-         }
-         tag_stats.update(size);
-         _dirty_tags.Add(tag);
-      }
+//       void _handle_publish(string tag, int size) {
+//          bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}");
+//          if (tag == tag_name)
+//             return;
+//          if (!_tag_stats_map.TryGetValue(tag, out var tag_stats)) {
+//             tag_stats = new TagStatsInfo(tag);
+//             _tag_stats_map.Add(tag, tag_stats);
+//          }
+//          tag_stats.update(size);
+//          _dirty_tags.Add(tag);
+//       }


-      #region IWorker Members
+//       #region IWorker Members

-      public void doWork() =>
-         workerImpl.doWork();
+//       public void doWork() =>
+//          workerImpl.doWork();

-      void doWorkImpl() {
-         if (_dirty_tags.Count > 0) {
-            MemoryStream str = new MemoryStream();
-            List<TagStatsInfo> dirty_ts_info = new List<TagStatsInfo>();
-            foreach (string tag in _dirty_tags) {
-               //_tag_stats_map[tag].write_to(str);
-               dirty_ts_info.Add(_tag_stats_map[tag]);
-            }
-            _dirty_tags.Clear();
-            var bytes = TagStatsInfo.serialize(dirty_ts_info);
-            PublishMsg pub_msg = new PublishMsg() {
-               connection = "Internal",
-               publishing_app = "PubSubServer",
-               tag = SystemStatsPublisher.tag_name,
-               tag_type = SystemStatsPublisher.tag_type,
-               publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-               data_to_publish = bytes
-            };
-            internalPublisher.Publish(pub_msg);
-         }
-      }
+//       void doWorkImpl() {
+//          if (_dirty_tags.Count > 0) {
+//             MemoryStream str = new MemoryStream();
+//             List<TagStatsInfo> dirty_ts_info = new List<TagStatsInfo>();
+//             foreach (string tag in _dirty_tags) {
+//                //_tag_stats_map[tag].write_to(str);
+//                dirty_ts_info.Add(_tag_stats_map[tag]);
+//             }
+//             _dirty_tags.Clear();
+//             var bytes = TagStatsInfo.serialize(dirty_ts_info);
+//             PublishMsg pub_msg = new PublishMsg() {
+//                connection = "Internal",
+//                publishing_app = "PubSubServer",
+//                tag = SystemStatsPublisher.tag_name,
+//                tag_type = SystemStatsPublisher.tag_type,
+//                publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+//                data_to_publish = bytes
+//             };
+//             internalPublisher.Publish(pub_msg);
+//          }
+//       }

-      #endregion
+//       #endregion

-      #region IPubSubServerTagHandler Members
+//       #region IPubSubServerTagHandler Members

-      public void update(string tag, byte[]? msg) { }
+//       public void update(string tag, byte[]? msg) { }

-      public byte[] get_content(string tag) {
-         var ser = TagStatsInfo.serialize(_tag_stats_map.Values);
-         return ser;
-         // MemoryStream str = new MemoryStream();
-         // BinaryWriter bw = new BinaryWriter(str);
-         // bw.Write(_tag_stats_map.Count);
-         // foreach (TagStatsInfo t in _tag_stats_map.Values)
-         //    t.write_to(bw.BaseStream);
-         // return str.ToArray();
-      }
+//       public byte[] get_content(string tag) {
+//          var ser = TagStatsInfo.serialize(_tag_stats_map.Values);
+//          return ser;
+//          // MemoryStream str = new MemoryStream();
+//          // BinaryWriter bw = new BinaryWriter(str);
+//          // bw.Write(_tag_stats_map.Count);
+//          // foreach (TagStatsInfo t in _tag_stats_map.Values)
+//          //    t.write_to(bw.BaseStream);
+//          // return str.ToArray();
+//       }

-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }
@@ -1,117 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using System.IO;
-using System.Threading;
-using mbts;
-using System.Threading.Tasks;
-using bts;
-using Bts.PubSub;
-using Bts.PubSub.Server;
-using Spark.Core;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using BCG.Utilities.General;
+// using System.IO;
+// using System.Threading;
+// using mbts;
+// using System.Threading.Tasks;
+// using bts;
+// using Bts.PubSub;
+// using Bts.PubSub.Server;
+// using Spark.Core;

-namespace BCG {
-   /// <summary>
-   /// This class publishes "System_Sub" to the PubSubServer using the 
-   /// internal_publish. The data associated with the "System_Sub" tag 
-   /// is the list fo all subscribers and publishers.
-   /// This class, SystemSubscriptionMapPublisher, gets the data by listening 
-   /// to PubsubState.onTagMapChanged.
-   /// It also serves as an IPubSubServerTagHandler, and resides in SingleTagData.
-   /// update calls are no-ops and on get_content it returns the total state
-   /// It does not have much state except for the dirty state. It builds the state to publish
-   /// as well as return in get_content, from calling into PububState
-   /// </summary>
-   class SystemSubscriptionMapPublisher : VirtualSingleton<SystemSubscriptionMapPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
-      private IPubSubState _state;
-      private HashSet<string> _dirty_tags = new HashSet<string>();
-      private readonly string _ip = SysInfoManager.getIP();
-      public static string tag_type = "System_Sub";
-      public static string tag_name = "System_AllSubscriptions";
-      readonly IInternalPublisher internalPublisher;
-      readonly IWorker workerImpl;
-      readonly object syncRoot = new();
+// namespace BCG {
+//    /// <summary>
+//    /// This class publishes "System_Sub" to the PubSubServer using the 
+//    /// internal_publish. The data associated with the "System_Sub" tag 
+//    /// is the list fo all subscribers and publishers.
+//    /// This class, SystemSubscriptionMapPublisher, gets the data by listening 
+//    /// to PubsubState.onTagMapChanged.
+//    /// It also serves as an IPubSubServerTagHandler, and resides in SingleTagData.
+//    /// update calls are no-ops and on get_content it returns the total state
+//    /// It does not have much state except for the dirty state. It builds the state to publish
+//    /// as well as return in get_content, from calling into PububState
+//    /// </summary>
+//    class SystemSubscriptionMapPublisher : VirtualSingleton<SystemSubscriptionMapPublisher>, IPubSubServerTagHandler, IParameterizedWorker {
+//       private IPubSubState _state;
+//       private HashSet<string> _dirty_tags = new HashSet<string>();
+//       private readonly string _ip = SysInfoManager.getIP();
+//       public static string tag_type = "System_Sub";
+//       public static string tag_name = "System_AllSubscriptions";
+//       readonly IInternalPublisher internalPublisher;
+//       readonly IWorker workerImpl;
+//       readonly object syncRoot = new();

-      public SystemSubscriptionMapPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
-         bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor");
-         this.internalPublisher = internalPublisher;
-         workerImpl = new SingleThreadedWorker(new AnonymousWorker(_internal_publish));
-         _state = state;
-         _state.onTagMapChanged += _on_tag_state_changes;
-      }
+//       public SystemSubscriptionMapPublisher(IPubSubState state, IInternalPublisher internalPublisher) {
+//          bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor");
+//          this.internalPublisher = internalPublisher;
+//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(_internal_publish));
+//          _state = state;
+//          _state.onTagMapChanged += _on_tag_state_changes;
+//       }

-      public string TagType => tag_type;
+//       public string TagType => tag_type;

-      public TimeSpan Interval => TimeSpan.FromMilliseconds(500);
+//       public TimeSpan Interval => TimeSpan.FromMilliseconds(500);

-      private void _on_tag_state_changes(string[] tags) {
-         Func<string> tags_to_string = () => {
-            var rv = String.Empty;
-            foreach (var t in tags) {
-               rv += t + " ";
-            }
-            rv.Trim();
-            return rv;
-         };
-         bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes tags: {tags_to_string()}");
-         lock (syncRoot) {
-            _dirty_tags.AddRange(tags);
-         }
-      }
+//       private void _on_tag_state_changes(string[] tags) {
+//          Func<string> tags_to_string = () => {
+//             var rv = String.Empty;
+//             foreach (var t in tags) {
+//                rv += t + " ";
+//             }
+//             rv.Trim();
+//             return rv;
+//          };
+//          bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes tags: {tags_to_string()}");
+//          lock (syncRoot) {
+//             _dirty_tags.AddRange(tags);
+//          }
+//       }

-      private void _internal_publish() {
-         lock (syncRoot) {
-            if (_dirty_tags.Count > 0) {
-               List<TagPubSubInfo> tpsis = new List<TagPubSubInfo>();
-               foreach (string tag in _dirty_tags) {
-                  Dictionary<string, string> subscribers, active_pubslishers;
-                  ICollection<Pair<string, string>> dead_publishers;
-                  _state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers);
-                  TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers,
-                                                      dead_publishers, _ip);
-                  tpsis.Add(ti);
-               }
-               _dirty_tags.Clear();
-               var ser_bytes = TagPubSubInfo.serialize(tpsis);
-               PublishMsg pub_msg = new PublishMsg() {
-                  publish_time_epoch_ms = 0,
-                  tag = SystemSubscriptionMapPublisher.tag_name,
-                  tag_type = SystemSubscriptionMapPublisher.tag_type,
-                  connection = "Internal",
-                  is_compressed = false,
-                  data_to_publish = ser_bytes
-               };
-               internalPublisher.Publish(pub_msg);
-            }
-         }
-      }
+//       private void _internal_publish() {
+//          lock (syncRoot) {
+//             if (_dirty_tags.Count > 0) {
+//                List<TagPubSubInfo> tpsis = new List<TagPubSubInfo>();
+//                foreach (string tag in _dirty_tags) {
+//                   Dictionary<string, string> subscribers, active_pubslishers;
+//                   ICollection<Pair<string, string>> dead_publishers;
+//                   _state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers);
+//                   TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers,
+//                                                       dead_publishers, _ip);
+//                   tpsis.Add(ti);
+//                }
+//                _dirty_tags.Clear();
+//                var ser_bytes = TagPubSubInfo.serialize(tpsis);
+//                PublishMsg pub_msg = new PublishMsg() {
+//                   publish_time_epoch_ms = 0,
+//                   tag = SystemSubscriptionMapPublisher.tag_name,
+//                   tag_type = SystemSubscriptionMapPublisher.tag_type,
+//                   connection = "Internal",
+//                   is_compressed = false,
+//                   data_to_publish = ser_bytes
+//                };
+//                internalPublisher.Publish(pub_msg);
+//             }
+//          }
+//       }

-      #region IWorker Members
+//       #region IWorker Members

-      public void doWork() =>
-         workerImpl.doWork();
+//       public void doWork() =>
+//          workerImpl.doWork();

-      #endregion
+//       #endregion

-      #region IPubSubServerTagHandler Members
+//       #region IPubSubServerTagHandler Members

-      public void update(string tag, byte[]? msg) { }
+//       public void update(string tag, byte[]? msg) { }

-      public byte[] get_content(string t) {
-         ICollection<string> all_tags = _state.all_tags;
-         List<TagPubSubInfo> ti_list = new List<TagPubSubInfo>();
-         foreach (string tag in all_tags) {
-            Dictionary<string, string> subscribers, activePubslishers;
-            ICollection<Pair<string, string>> deadPublishers;
-            _state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
-            TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip);
-            ti_list.Add(ti);
-         }
-         var rv = TagPubSubInfo.serialize(ti_list);
-         return rv;
-      }
+//       public byte[] get_content(string t) {
+//          ICollection<string> all_tags = _state.all_tags;
+//          List<TagPubSubInfo> ti_list = new List<TagPubSubInfo>();
+//          foreach (string tag in all_tags) {
+//             Dictionary<string, string> subscribers, activePubslishers;
+//             ICollection<Pair<string, string>> deadPublishers;
+//             _state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
+//             TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip);
+//             ti_list.Add(ti);
+//          }
+//          var rv = TagPubSubInfo.serialize(ti_list);
+//          return rv;
+//       }

-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }
@@ -13,46 +13,70 @@ using Bts.PubSub.Server;
 using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Hosting;
 using System.Threading.Tasks;
+using BlueOrb.PubSub.Server;
+using System.Linq;

 namespace BCG {
   // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it.
   // The only outward facing members of this class is start
   // It has an IPubsubMessageHub, that it gets messages from, and updates the PubSubState (which really is all the pubsub server state)
-   class PubSubMessageReactor : IHostedService {
+   public class PubSubMessageReactor : IHostedService {
      private IPubsubMessageHub _msg_hub;
      // private delegate void MyMsgHandler(string sender, Stream msg, MetaData m);
      // private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>();
      private Thread _msg_polling_thread;
      // all the data,list of subscribers etc are in the state
      private IPubSubState _state;
-      private IInternalPublisherQueue _internal_pub_queue;
+      private List<IPubSubInternalPublisher>? _internal_publishers;
      private Stopwatch _stop_watch = new Stopwatch();
      TaskCompletionSource? executionTask;
      CancellationTokenSource? executionCts;

-      public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, IOptions<PubSubConfig> config)
-         : this(state, internalPublisherQueue, config.Value) { }
+      // public PubSubMessageReactor(IPubSubState state, IOptions<PubSubConfig> config)
+      //    : this(state, null, config.Value) {
+      //    bts.Log.Gen.info("PubSubMessageReactor.cons 1 System Initialized");
+      // }

-      public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, PubSubConfig config)
+      public PubSubMessageReactor(IPubSubState state, IEnumerable<IPubSubInternalPublisher>? internalPublishers, PubSubConfig config)
         : this(
            state,
-            internalPublisherQueue,
+            internalPublishers,
            config.ServerPort.assert_value(),
            TimeSpan.FromMilliseconds(config.HeartbeatInterval.assert_value()),
            config.MissingHeartbeatsBeforeGc.assert_value(),
-            TimeSpan.FromMilliseconds(config.DisposedConnectionCheckPeriod.assert_value())) { }
+            TimeSpan.FromMilliseconds(config.DisposedConnectionCheckPeriod.assert_value())) {
+         bts.Log.Gen.info("PubSubMessageReactor.cons 2 System Initialized");
+      }

-      public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval) {
-         bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
+      // public PubSubMessageReactor(IPubSubState state, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval)
+      //    : this(state, null, port, heartbeatInterval, maxMissedHeartbeats, collectDisposedConnectionsInterval) {
+      //    bts.Log.Gen.info("PubSubMessageReactor.cons 3 System Initialized");
+      // }
+
+      public PubSubMessageReactor(IPubSubState state, IEnumerable<IPubSubInternalPublisher>? internalPublishers, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval) {
+         bts.Log.Gen.info("PubSubMessageReactor.cons 4 System Initialized");

         _state = state;
-         _internal_pub_queue = internalPublisherQueue;
         _msg_polling_thread = new Thread(new ThreadStart(_polling_thread_func));
         _msg_polling_thread.Priority = ThreadPriority.AboveNormal;
         _msg_hub = new ManagedTCPMessageHub(port, heartbeatInterval, maxMissedHeartbeats, collectDisposedConnectionsInterval);
         // this is the only event this class gets from hub
         _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection));

+         _internal_publishers = internalPublishers is null
+            ? null
+            : new List<IPubSubInternalPublisher>(internalPublishers);
+         //Log.Gen.info($"internalPublishers: {_internal_publishers.First().GetType()}");
+         _internal_publishers?.ForEach(publisher => publisher.onNewData += (tag, tag_type, data) => {
+            PublishMsg pub_msg = new PublishMsg() {
+               connection = "Internal;",
+               tag = tag,
+               tag_type = tag_type,
+               data_to_publish = data
+            };
+            _handle_pub("Internal", pub_msg, true);
+         });
+
      }

      // the connection callback is the only event we get back from the message_hub.
@@ -89,29 +113,17 @@ namespace BCG {
      /// </summary>
      private void _polling_thread_func() {
         Log.Gen.info("PubSubMessageReactor._polling_thread_func starting thread function _polling_thread_func");
-         TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500);
         _msg_hub.activate();
-         Action handle_internal_pubs = () => {
-            Log.Gen.info($"PubSubMessageReactor._polling_thread_func handle_internal_pubs");
-            var internal_pubs = _internal_pub_queue.DequeueMultiple(TimeSpan.FromSeconds(1), 64000);
-            //Log.Gen.info($"PubSubMessageReactor._polling_thread_func internal_pubs count: {internal_pubs.Count}");
-            foreach (var msg in internal_pubs) {
-               try {
-                  _handle_pub("Internal", msg, true);
-               } catch (Exception ex) {
-                  bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}");
-               }
-            }
-         };
-
         while (executionCts is null || !executionCts.IsCancellationRequested) {
            try {
+               foreach (var w in _internal_publishers) {
+                  w.doWork();
+               }
               //bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
               var received_data = _msg_hub.get_all_updates();
               var n_msgs = received_data.Count;
               if (n_msgs == 0) {
-                  handle_internal_pubs();
-                  System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
+                  System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(10));
                  continue;
               }
               bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
@@ -119,9 +131,10 @@ namespace BCG {
                  // multiple pubs for the same tags should be batched here
                  ArgumentException.ThrowIfNullOrEmpty(kv.first);
                  ArgumentNullException.ThrowIfNull(kv.second);
+                  var stop_watch = Stopwatch.StartNew();
                  _handle_client_msg(kv.first, kv.second);
+                  bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func _handle_client_msg sender: {kv.first} processing time: {stop_watch.ElapsedMilliseconds} msecs");
               }
-               handle_internal_pubs();
            } catch (Exception ex) {
               bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}");
               if (ex is OutOfMemoryException) {
@@ -209,7 +222,7 @@ namespace BCG {
      }
      private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) {
         bts.Log.assert(!pub_msg.is_compressed);
-         //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}");
+         bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}");
         // this is where pubsub's data state is updated
         _state.update_pub_data(pub_msg);
         ArgumentNullException.ThrowIfNull(pub_msg.tag);
@@ -1,11 +1,12 @@
-using System.Text;
+using System.Text;
 using System.IO;
+
 namespace BCG;

 /// <summary>
 /// This class represents an event happening to a single tag in a pubsub (or substation).
 /// It could be that a tag is being published, or it could be a subscription to a tag or
-/// an unsubscription to a tag. 
+/// an unsubscription to a tag.
 /// </summary>
 public class PubSubSingleTagStateEvent {
   public PubSubSingleTagStateEvent(string tag, EventType eventType, string connection, string app) {
@@ -32,7 +33,6 @@ public class PubSubSingleTagStateEvent {
      return fromStream(mem);
   }

-   //for debugging
   public override string ToString() {
      var bd = new StringBuilder();
      bd.Append(tag + " ");
@@ -49,7 +49,6 @@ public class PubSubSingleTagStateEvent {
         case EventType.unsubscription:
            bd.Append("was unsubscribed to");
            break;
-
      }
      bd.Append(" app: " + app);
      bd.Append(" connection: " + connection);
Author	SHA1	Message	Date
Taha	7cff8b625d	Cleaning market data streams	2026-06-09 22:58:04 -04:00
Taha	6950de82ff	Replay streams + TC2 position	2026-04-21 14:46:20 -04:00
Taha	9140888b54	launch	2026-04-13 23:46:47 -04:00
Taha	32a851954c	BlueOrb signals	2026-04-13 23:46:21 -04:00
John Colvin	54c97b5610	fix: Set buffer Position to 0 when deserializing pubsub echo payload	2026-02-21 16:36:53 -05:00
John Colvin	5c05afd9b3	fix: Only fire stream update if there are actually changes In other words, don't fire if addedRows and deletedRows are both empty.	2026-02-21 16:36:53 -05:00
John Colvin	dcbba39e4a	fix: Add null check in BlueDataUtility.build_dp_inc	2026-02-21 16:36:53 -05:00
John Colvin	d09f22007e	fix: Ensure PubSubTagHandlerFactory uses correct config * Add this service as part of the PubSubClient services for dep injection (it is needed by PublisherCore) * Get rid of PublisherCore's default constructor, so dep injection treats the tag handler factory as a required param (fails if it isn't available) - Replace the default ctr with a static "Create" fn, so code that hasn't been updated to use dependency injection can still use that. - Use PubSubConfig.Instance (runtime) instead of compile time Default * Fix compile time Default config - In fallback handler config, change Tag from "" to ".*" and move it from the beginning of the list to the end (it should be the last resort, not the other way around).	2026-02-21 16:36:53 -05:00
John Colvin	a07e7ea6b8	fix: Don't call MasterWorker/vsd doWork on DSQA ui thread Now that we use dependency injection, this is handled by a dedicated service (MasterWorkerService). As a side effect, stream updates aren't necessarily fired on the ui thread, so we refresh the grid on a form timer callback.	2026-02-21 16:36:47 -05:00
John Colvin	4da52ff724	fix: Replace DataPacket w/ IDataPacket in DSQA	2026-02-21 14:49:39 -05:00
John Colvin	05c7a32fc5	refactor: Add parameterless ctr to PubsubEchoTestStream So it's quick/easy to check in DSQA (less typing required)	2026-02-21 14:23:51 -05:00
John Colvin	404d3073f9	ui: Unhide PubsubHeartbeatStream (pss_hb) from DSQA	2026-02-21 14:11:46 -05:00
John Colvin	7f1154d70c	cleanup: Fix indentation in PubsubHeartbeatStream.cs	2026-02-21 14:10:11 -05:00
John Colvin	5bc2e29096	fix build: Only check "dsqa" flag when building on windows The --dsqa opt isn't defined when running on linux.	2026-02-21 17:55:12 +00:00
John Colvin	f5731a184c	fix: Actually disable Spark in default DSQA config	2026-02-20 11:08:33 -05:00
John Colvin	120fe8e5c6	vscode: Add DSQA debug config to skip build	2026-02-20 10:44:19 -05:00
John Colvin	137fdc537d	refactor: Move DSQA config to appsettings.json	2026-02-20 10:43:13 -05:00
John Colvin	a605e57bee	build: Add dsqa toggle to build_data_stream.py Windows only	2026-02-20 02:33:38 -05:00
John Colvin	3ca4bf784c	ui: Remove bts.BlueOrb namespace filter for DSQA streams	2026-02-20 02:13:17 -05:00
John Colvin	6c3b7ebdaf	refactor: Use dependency injection in DSQA	2026-02-20 02:13:17 -05:00
John Colvin	2fbb6fe563	Disable Spark in default DSQA config	2026-02-20 02:13:17 -05:00
John Colvin	9ea33ac110	ui: Unhide PubSubServerStateStream (pss_state) from DSQA	2026-02-20 02:13:17 -05:00
John Colvin	0bc021912a	fix: Replace more DataPacket references with IDataPacket	2026-02-20 02:13:17 -05:00
John Colvin	fa017bd9ff	fix build: Remove old ViewStream.rows references from dsqa The 'rows' prop no longer exists	2026-02-20 02:13:17 -05:00
John Colvin	57b59215e6	fix build: Don't echo parenthesis in .cmd if block It's not allowed (because windows).	2026-02-18 02:54:25 -05:00
arham	210768e391	fix: handle badrequests in websocket api	2026-02-18 05:40:27 +00:00
arham	f01e8a578d	Add: enter synthetic spreader orders through	2026-02-18 05:36:02 +00:00
arham	528a00cf98	Add: config for okx_bin_sim	2026-02-18 05:31:18 +00:00
arham	96a04a7c84	Fix: side and status to json in orderrecodtuple	2026-02-18 05:30:28 +00:00
arham	abaf48beb4	Add: other fields to tosjon for security.cs	2026-02-18 05:29:03 +00:00
John Colvin	08fcae94c0	Update externals (moved emumba deribit deb to bitbucket cloud)	2026-02-17 18:16:22 +00:00
John Colvin	30814a2ed6	fix build: Fix Bts.Extensions.Hosting projref path	2026-02-15 16:48:02 +00:00
John Colvin	3c61de5f13	devcontainer: Install Rakurai deb Download from bitbucket cloud.	2026-02-14 18:43:07 +00:00
John Colvin	771c58d4ad	devcontainer: Move emumba deb to bitbucket cloud I just learned that Bitbucket offers per-repo cloud storage for arbitary files (the "Downloads" section on the web gui). Doing this before adding another third-party deb to our build (rakurai), which is quite large at 140MB. We don't want to pack that into our repo.	2026-02-14 18:42:03 +00:00
John Colvin	5db5d40a99	devcontainer: Fix CMake NO_PUBLIC_KEY warning	2026-02-14 18:29:28 +00:00
John Colvin	020ec9c6bf	build: Replace proj refs w/ internal nugets when packing	2026-02-03 16:57:32 +00:00
John Colvin	4e28835bbd	vscode: Add tools/py to shell PYTHONPATH	2026-02-03 16:57:32 +00:00
John Colvin	6985fbb090	refactor: Move CompressionUtilities into separate csproj It was nested in PubSub. The new csproj is named Bts.IO.Compression.	2026-02-03 16:57:32 +00:00
John Colvin	2f42311220	build: Update a few python packages * protobuf 4.21.12 -> 6.33.4 * pybind11 2.13.6 --> 3.0.1 * pybind11-stubgen 2.5.3 ---> 2.5.5	2026-02-03 16:57:32 +00:00
John Colvin	3842efcef1	build: Define build py requirements in separate files No need to install backend build deps when building the front-end, and vice versa. The build scripts (build_spark.cmd, build_sparkd.sh, etc.) automatically install only the necessary requirements now. Note: We'll keep the monolith requirements.txt file around (at least for now), as it includes more than just build deps (e.g., numpy, pandas, and jupyter for data analysis; pybit for bybit plugins, etc.).	2026-02-03 16:57:32 +00:00
John Colvin	635107eb68	cleanup: Remove unused py code	2026-02-03 05:20:29 +00:00
John Colvin	1b9a49b924	cleanup: Add missing files and env vars to sparkd config profiles	2026-02-03 05:20:29 +00:00
John Colvin	ead35389d5	build: Make PubSub packable	2026-02-03 05:20:29 +00:00
John Colvin	22b44845fb	build: Update Microsoft.AspNetCore.Cors to 2.3.9	2026-02-03 05:20:29 +00:00
John Colvin	8ab26f3e1d	build: Remove dependency on Newtonsoft.Json Replace with System.Text.Json (official dotnet)	2026-02-03 05:16:21 +00:00
John Colvin	01c61bb16d	fix build: Publish spark_api_server csproj instead of the sln There's a version conflict with System.Data.Common when publishing the sln.	2026-01-22 05:50:49 +00:00
John Colvin	e09e49ce56	fix build: Add Bts.Net to Spark.Api.Server.sln It seems like the dep is automatically detected on windows, but not linux (so we need to add it explicitly).	2026-01-21 15:06:00 -05:00
John Colvin	e6cc20858b	build: Honor "UseProjectRefs" in spark_websocket, spark_api_server	2026-01-21 14:28:48 -05:00
John Colvin	2eb80082d8	refactor: Move Bts.Extensions.Hosting, Bts.Net to cs dir They aren't exclusively used by DataStream csprojs	2026-01-21 14:20:39 -05:00
John Colvin	d55ef854ba	fix build: Don't actually copy files when building with --dry-run	2026-01-21 14:03:01 -05:00
John Colvin	b1417ece5d	fix build: Fix cmake windows preset in publish_api_server.py Changed from 'spark' -> 'spark-windows'	2026-01-21 14:03:01 -05:00
John Colvin	f6ae219a72	build: Add a few toggle opts to publish py scripts --(no-)cpp Enable/disable cpp build --(no-)cs Enable/disable csharp build --(no-)csproj-refs Depend on csproj refs, or locally built nugets	2026-01-21 14:03:01 -05:00
John Colvin	1e4b9cf833	Add new StartWithOnTimeout ext fn for IObservable Ophir wanted this functionality for data in the sdk	2026-01-19 10:44:51 -05:00
John Colvin	228931dd2a	cleanup: Fix indentation in launch.json Also rename launch task "(gdb) Attach" -> "Sparkd Attach"	2026-01-19 10:44:51 -05:00
John Colvin	f5b4b36ea1	Move devcontainer Dockerfile to bluetradingsystems/linux_dotnet8_dev We now host it on dockerhub under that name	2026-01-19 10:44:51 -05:00
arham	f19d2d695b	Fix: Updated path so that profiles work	2026-01-16 06:55:09 +00:00
John Colvin	d4b9b6bcba	fix: Dispose subscription in TopOfBookQuoter demo	2026-01-14 10:35:42 -05:00
John Colvin	84d74a7873	refactor: Actually implement IDisposable in MarketDataClient	2026-01-14 10:35:42 -05:00
John Colvin	8c34280d75	fix: Init PriceTicker Security to avoid null ref in Equals impl	2026-01-14 10:35:42 -05:00
John Colvin	474d96ced4	feat: Add PriceStats/Ticker to ISimpleSparkClient (cs sdk)	2026-01-14 10:35:42 -05:00
John Colvin	1432692a26	refactor: Make PriceStats/Ticker record structs Also rename props in PascalCase	2026-01-13 13:57:47 -05:00