refactor: Add new glbx variable ticks to security (shm and proto)

diag: Add logs to MDP3Listener::onOption
refactor: Enable nullability in Bts.Spark.Api
2026-03-04 08:00:40 +00:00 · 2026-03-03 15:28:13 +00:00 · 2026-03-03 15:06:06 +00:00 · 2026-03-03 15:06:06 +00:00 · 2026-03-03 15:06:06 +00:00 · 2026-03-03 15:06:06 +00:00
651 changed files with 3687 additions and 25372 deletions
@@ -29,9 +29,6 @@
 #     tools/deploy.sh --help
 #

-### Host/Runtime ###
-TZ=Etc/UTC
-
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -41,9 +38,6 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
-# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
-# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
-# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -51,8 +45,6 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
-SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh

 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
@@ -183,10 +183,11 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
      "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -213,7 +214,7 @@
      "type": "coreclr",
      "request": "launch",
      "program": "",
-      "preLaunchTask": "Build Spark Sdk (Debug)",
+      "preLaunchTask": "Build Api Server (Debug)",
      "windows": {
        "program": "Bts.Spark.Sdk.Demo.exe",
      },
@@ -285,7 +286,7 @@
      "windows": {
        "program": "Spark.exe",
        "args": [
-          "--Spark:ConfigServer",
+          "--sparkd",
          "${input:sparkConfigServer}"
        ],
        "cwd": "build/Debug/bin/net8.0-windows",
@@ -448,15 +449,14 @@
      "default": "Local",
    },
    {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
      "type": "pickString",
-      "description": "PubSubServer Configuration",
+      "description": "PubSub Configuration",
      "options": [
-        "Development",
-        "EchoServer",
-        "Local",
+        "PubSubServer",
+        "SimpleEchoClient",
      ],
-      "default": "Development",
+      "default": "PubSubServer",
    },
    {
      "id": "sdkDemoEnvironment",
@@ -17,6 +17,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{3BF4DF96-5AFF-4674-866A-71A8090627FF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{62067659-AA2E-4CB5-B7CE-0B38EAA92080}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -87,6 +95,54 @@ Global
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.Build.0 = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.Build.0 = Debug|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.ActiveCfg = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.Build.0 = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.ActiveCfg = Release|Any CPU
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.Build.0 = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.Build.0 = Debug|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.Build.0 = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.ActiveCfg = Release|Any CPU
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.Build.0 = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.Build.0 = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.Build.0 = Debug|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.Build.0 = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.ActiveCfg = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.Build.0 = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.ActiveCfg = Release|Any CPU
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.Build.0 = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.Build.0 = Debug|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.Build.0 = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.ActiveCfg = Release|Any CPU
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -98,5 +154,9 @@ Global
 		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{3BF4DF96-5AFF-4674-866A-71A8090627FF} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{62067659-AA2E-4CB5-B7CE-0B38EAA92080} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal
@@ -51,6 +51,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{802ECF0B-7264-4168-95D1-158E47FB65CC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{0354655D-2394-42BB-B1D4-2D75B9BCABF1}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -571,6 +579,102 @@ Global
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.ActiveCfg = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.Build.0 = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.ActiveCfg = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.Build.0 = Release|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.ActiveCfg = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.Build.0 = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.ActiveCfg = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.Build.0 = Release|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.ActiveCfg = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.Build.0 = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.ActiveCfg = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.Build.0 = Release|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.ActiveCfg = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.Build.0 = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.ActiveCfg = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.Build.0 = Release|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -592,6 +696,10 @@ Global
 		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{802ECF0B-7264-4168-95D1-158E47FB65CC} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{0354655D-2394-42BB-B1D4-2D75B9BCABF1} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
@@ -2,7 +2,6 @@ services:
  sparkd:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
@@ -10,35 +9,24 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-shell:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-    build:
-      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: "do sleep 1; done"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
@@ -54,7 +42,6 @@ services:
  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
@@ -63,8 +50,26 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
+        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
+          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/.local/share/bts/sparkd/conf/
+        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
+        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
+          set -a
+          . /home/bts/.local/share/bts/sparkd/conf/.env.local
+          set +a
+        fi
+        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
+          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
+        fi
+        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
@@ -72,7 +77,6 @@ services:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
@@ -80,65 +84,7 @@ services:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
-  sparkd-multicast:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
-  
-  sparkd-multicast-cert:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
+
  sparkd-tail:
    image: bluetradingsystems/linux_dotnet8
    volumes:
@@ -249,6 +195,3 @@ volumes:
    name: "md_publisher_22019"
  historic_md:
    name: "historic_md"
-  md_recording:
-    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
-    external: true
@@ -1,942 +0,0 @@
-# Security ID handling in `sparkd`
-
-This document traces how security identifiers are created, translated, persisted,
-indexed, and published by the `sparkd` market-data paths. It focuses on the
-legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
-
-## Executive summary
-
-`sparkd` uses two identifiers that must not be confused:
-
-| Identifier | Meaning | Typical field | Lifetime |
-|---|---|---|---|
-| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
-| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
-
-`invalid_security_key` is `0`; a security with key `0` is not yet publicly
-discoverable through the shared-memory consumer path.
-
-The two principal paths differ:
-
- **MDP3/OnixS:** the definition callback sets only the CME ID in
-  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
-  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
-  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
-  calls `MDProducerHandle::set_key()`.
- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
-  currently uses the ID already present on the incoming domain security
-  (`use_given_sec_key == true`). Live adapters normally obtain that ID
-  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
-  it to both `FixedSecurity::key` and `numeric_id`, so
-  `MDProducer::add_security()` publishes it synchronously. Only the listener's
-  own fallback `IDGenerator` call is bypassed by the current constant.
-
-## End-to-end diagrams
-
-### CME MDP3 / OnixS
-
-```mermaid
-sequenceDiagram
-    participant FE as OnixS feed-engine thread
-    participant L as MDP3Listener
-    participant P as MDProducer / shared memory
-    participant Q as SecurityIDRecorder queues
-    participant D as sparkd dispatcher thread
-    participant DB as key_ids.sqlite
-
-    FE->>L: instrument definition (CME SecurityID)
-    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
-    L->>P: add_security(FixedSecurity)
-    P->>P: allocate hot + cold entries with key 0
-    P->>Q: ensure_key(handle)
-    L->>L: index[CME SecurityID] = handle
-    Note over FE,L: Books can route by CME ID while the Spark key is pending
-    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
-    Q->>DB: find (namespace, descriptor), or insert a new row
-    DB-->>Q: SQLite rowid = Spark security key
-    Q->>P: handle.set_key(rowid)
-    P->>P: hot.key first, then cold security.key, then increment security_count
-    Note over P: The security is now public to shared-memory consumers
-```
-
-### Generalized/newer pipeline (current behavior)
-
-```mermaid
-sequenceDiagram
-    participant A as Venue adapter / plugin
-    participant DB as IDGenerator / key_ids.sqlite
-    participant G as GeneralMarketDataListener
-    participant I as GeneralSecurityIndex
-    participant P as MDProducer / shared memory
-
-    A->>DB: get_or_add(venue, type, adapter key)
-    DB-->>A: SQLite rowid as domain SecurityID
-    A->>G: on_securities(domain Security list with ID)
-    G->>G: _find_sec() - use incoming security_common.id
-    G->>G: _copy_security_common() - key = id, numeric_id = id
-    G->>P: add_security(FixedSecurity with non-zero key)
-    P->>P: set_key() during add_security()
-    G->>I: index[id] = handle
-    Note over G,P: No asynchronous ID assignment in the current configuration
-```
-
-## Data-flow charts
-
-The sequence diagrams above emphasize timing and thread transitions. These
-flowcharts emphasize which identifier is carried through each representation
-and where data is retained.
-
-### MDP3 / legacy data flow
-
-```mermaid
-flowchart TD
-    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
-
-    ONIXS --> DEF["MDP3Listener definition callback"]
-    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
-
-    FIXED --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
-    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
-
-    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
-
-    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
-    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
-    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
-    UNRESOLVED --> CMEINDEX
-    CMEINDEX --> LEGS
-    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
-
-    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
-    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
-    INSERT --> ROWID
-
-    ROWID --> SETKEY["MDProducerHandle::set_key"]
-    SETKEY --> HOT["Write hot-entry key"]
-    HOT --> COLD["Write cold security key"]
-    COLD --> COUNT["Increment security_count"]
-    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
-
-    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
-    UPDATE --> CMEINDEX
-    CMEINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> LIVE["Update live shared-memory data"]
-    PUBLIC --> LIVE
-```
-
-The CME index is usable before the Spark key is assigned, so feed updates can
-reach the internal handle while the security is still hidden from consumers.
-The persistent rowid crosses the publication boundary only at `set_key()`.
-
-### Generalized/newer data flow
-
-```mermaid
-flowchart TD
-    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
-
-    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
-    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
-    IDINPUT --> MUTEX["IDGenerator global mutex"]
-    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid"]
-    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
-    INSERT --> ROWID
-    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
-    IDTYPE -- Recorded or pre-keyed --> DOMAINID
-
-    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
-    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
-    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
-    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
-
-    CONVERT --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
-    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
-    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
-
-    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
-    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
-    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
-    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
-    PYHOT --> EVENTID
-    PYHOT -. uses same mutex and SQLite path .-> MUTEX
-    EVENTID --> CCMAP
-    CCMAP --> GENERALINDEX
-    GENERALINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
-    SHM --> SHMUPDATE
-
-```
-
-In the normal newer live path, persistence occurs before the definition enters
-the generalized listener. The DB call therefore adds synchronous adapter-side
-latency, but publication needs no later timer. Most adapters use an in-memory
-map for subsequent events; the dotted Python path is the documented exception.
-
-## The publication boundary
-
-`MDProducerHandle::set_key()` is the definitive publication point for the
-legacy shared-memory representation. Its write ordering is intentional:
-
-1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
-2. Write the key into the cold security entry.
-3. Increment the cold header's public `security_count`.
-
-The cold security key is written last because consumers treat a non-zero cold
-key as visibility. At that moment the corresponding hot entry must already be
-addressable. `set_key()` only permits the transition from `0` to one key; it is
-not a key-update mechanism.
-
-## Identifier representations
-
-The names are historically inconsistent. In particular, many protobuf fields
-named `security_id` contain the **Spark key**, not the exchange's ID.
-
-| Layer | Exchange/feed identifier | Spark identifier |
-|---|---|---|
-| MDP3 wire/OnixS | `securityId()` | not present |
-| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
-| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
-| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
-| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
-| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
-| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
-| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
-
-The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
-shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
-64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
-allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
-no explicit checked conversion.
-
-## Which parts use exchange IDs, descriptors, or Spark security IDs?
-
-The three identifier classes serve different jobs:
-
- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
- **Descriptor/natural key:** recognizes a definition logically at the
-  persistence, discovery, or human-facing boundary.
- **Spark security ID/key:** joins the instrument to books, orders, fills,
-  positions, and services after publication.
-
-The core downstream system does not repeatedly identify a security from its
-description. The description is primarily used to obtain a Spark key; after
-that boundary, most joins use the numeric Spark key.
-
-### Component-level usage matrix
-
-| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
-|---|---|---|---|
-| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
-| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
-| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
-| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
-| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
-| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
-| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
-| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
-| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
-| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
-| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
-| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
-| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
-| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
-| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
-| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
-| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
-| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
-| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
-| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
-
-### Exchange/external IDs: where they stop
-
-In MDP3 the separation is explicit:
-
-```text
-CME SecurityID
-    -> FixedSecurity.numeric_id
-    -> CMESecurityIndex lookup
-    -> MDProducerHandle
-    -> Spark security_key for downstream publication
-```
-
-The external CME ID continues to route feed messages but does not become the
-order/position key. This permits the same downstream Spark key to survive a
-restart as long as the logical descriptor mapping remains in
-`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
-
-The generalized path collapses more of these roles:
-
-```text
-exchange/vendor identity
-    -> adapter calls IDGenerator
-    -> domain::SecurityID
-    -> FixedSecurity.key and FixedSecurity.numeric_id
-    -> GeneralSecurityIndex and downstream Spark key
-```
-
-`SecurityCommon::exchange_sec_id` may still contain the original textual
-external identifier, but `GeneralMarketDataListener` does not use it to route
-events and does not put it in `numeric_id`. The adapter must translate later
-events to `domain::SecurityID` before invoking the listener.
-
-### Descriptor/name users
-
-Descriptors are used in three main places:
-
-1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
-   descriptor, while `IDGenerator` uses its caller-provided `name`.
-2. **Definition lookup and discovery:** `SecurityCache` supports
-   venue/type/name lookup; symbol filters and administrative tools use security
-   metadata to select or display instruments.
-3. **Diagnostics:** logs include symbols and descriptions so humans can
-   understand a numeric key.
-
-Descriptions generally do not drive the high-rate book, trade, order, fill, or
-position paths. Those paths use numeric IDs after definition resolution.
-
-It is important that “descriptor” does not mean one universal canonical field:
-
- Legacy CME uses Globex `security_desc` from tag 55.
- Many generalized adapters use `sec.desc()` or an exchange instrument name.
- Databento uses its numeric instrument ID converted to a string.
- Python uses `exchange_security_id`.
-
-Therefore, two adapters may describe the same economic contract but request
-different logical rows and receive different Spark IDs.
-
-### Spark security-ID users
-
-Once a security is published, its Spark key is the dominant system identifier:
-
-```mermaid
-flowchart LR
-    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
-    KEY --> MDC["MDConsumer and book subscriptions"]
-    KEY --> MBO["MBO snapshots and queries"]
-    KEY --> OMS["Orders, algorithms, and spread legs"]
-    KEY --> FILLS["Fills and execution records"]
-    KEY --> POS["Positions, PNL, and risk"]
-    KEY --> API["Services and client APIs"]
-    KEY --> REC["Relay and market-data recordings"]
-    KEY --> SECDB["Serialized security definitions"]
-```
-
-Fields named `security_id`, `security_key`, or top-level `Security.id()` in
-these downstream areas almost always mean the Spark key. They should not be
-compared directly with CME `SecurityID`, Databento `instrument_id`, or another
-venue/vendor identifier without going through the relevant adapter/index
-mapping.
-
-## Persistence and allocation
-
-### The authoritative key mapping: `key_ids.sqlite`
-
-Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
-file is:
-
-```text
-<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
-```
-
-The database contains one table and two indexes:
-
-```sql
-CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
-CREATE UNIQUE INDEX key_index ON store(ns, key);
-CREATE INDEX ns_index ON store(ns);
-```
-
-The SQLite `rowid` is the generated Spark key. `ns` is normally
-`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
-descriptor chosen by the caller. `value` is normally empty. Therefore ID
-stability depends on all three of these remaining stable:
-
- the configured `data_path` and preservation of this database;
- the venue/type namespace;
- the descriptor supplied by the feed adapter.
-
-The two allocator implementations share the file and convention:
-
- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
-  uses `securities::descriptor()`. For CME this is the Globex
-  `security_desc`, populated from the definition's symbol/tag 55.
- `IDGenerator` (newer): derives the namespace from new domain enums and uses
-  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
-  identifier; Databento deliberately passes the decimal instrument ID, and the
-  Python bridge uses the exchange security ID.
-
-### Logical-row examples
-
-The following names are illustrative. Production uses the exact string emitted
-by the selected feed/adapter, including its case, spacing, and weekly-series
-notation.
-
-#### Legacy `SecurityIDRecorder`
-
-For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
-Globex `security_desc`. After converting the fixed security to
-`api::Security`, `securities::descriptor()` returns that same
-`glbx.security_desc()` string.
-
-| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
-|---|---|---|---|
-| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
-| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
-| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
-| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
-
-A weekly option does not get a special namespace. Standard and weekly options
-are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
-`security_desc` distinguishes the weekly series. For example, if the database
-assigned rowids `42001` through `42004` to the rows above, those integers—not
-the descriptor strings—would be the Spark security keys.
-
-Conceptually, legacy allocation is:
-
-```text
-future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
-option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
-weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
-spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
-```
-
-#### Newer `IDGenerator`
-
-The namespace result is similar, but the third argument is chosen by the
-adapter. Examples using a description-oriented adapter are:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
-// store: ns="GLBX.FUTURE", key="ESZ6"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6 C6500"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
-
-get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
-// store: ns="COINBASE.SPOT", key="BTC-USD"
-```
-
-Databento intentionally produces a different logical row even when the human
-description is `ESZ6`. If its instrument ID were `12345678`, it would call the
-equivalent of:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
-// store: ns="GLBX.FUTURE", key="12345678"
-```
-
-Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
-`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
-while a provider that sends `exchange_security_id="BTCUSD"` produces a
-different row and therefore a different Spark key.
-
-These examples show why the tuple is only logical when the adapter's naming
-contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
-describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
-
-Both enum-prefix removals currently produce the same namespace spelling (for
-example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
-rowid space. Live rows cannot share a rowid. The table does not declare
-`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
-SQLite to reuse a number later. Security-key rows should therefore be treated
-as append-only operational data.
-
-`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
-and feed latency over maximum durability against a host/power failure. The
-unique `(ns, key)` index prevents two different rows for the same logical
-security.
-
-### Other places the key is stored
-
-| Location | Purpose | Authoritative for allocation? |
-|---|---|---|
-| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
-| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
-| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
-| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
-| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
-
-The `SecurityIDRecorder::work_db()` comment says a published security is also
-recorded in the security database, but that function does **not** write
-`securities.sqlite` in the current code. Full security definitions are persisted
-separately by `SecurityCache::flush_to_disk()` through
-`db::insert_or_replace()`.
-
-## CME MDP3 / OnixS pipeline in detail
-
-### 1. Construction and thread setup
-
-`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
-`MDProducer`. It constructs the recorder with:
-
- a lookup callback that maps `ExchangeKey` through the CME index to the
-  handle's current Spark key; and
- `Dispatcher::current()`, which drives a 200 ms recorder timer.
-
-In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
-dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
-then calls that thread's dispatcher `run()`. The older `md run onixs` command
-uses its command/dispatcher thread similarly.
-
-The OnixS feed engine has its own configured thread pool. Instrument and market
-data callbacks therefore originate on OnixS feed-engine threads, while recorder
-database work and final key assignment execute on the MDP3 dispatcher thread.
-
-| MDP3 stage | Executing thread | DB call? |
-|---|---|---|
-| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
-| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
-| Add CME-ID index entry | Same OnixS callback thread | No |
-| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
-| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
-| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
-
-### 2. Definition callback: exchange ID is set, Spark key is not
-
-For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
-
-```cpp
-fsec.numeric_id = def.securityId();
-```
-
-It also copies symbol/descriptor/venue and pricing fields. A newly constructed
-`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
-does not change it.
-
-For a spread, every leg starts with:
-
-```cpp
-leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
-leg.key = invalid_security_key;
-```
-
-Thus both the parent and legs initially carry exchange-routing identity, but no
-Spark identity.
-
-### 3. Shared-memory allocation and exchange indexing
-
-`MDP3Listener::add_security()` performs these operations in order:
-
-1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
-   cold entry and copies the security into the cold segment.
-2. Because the key is `0`, `MDProducer` queues the handle through
-   `SecurityIDRecorder::ensure_key()`.
-3. `_index.add_security(security.numeric_id, handle)` immediately creates the
-   CME-ID-to-handle mapping.
-4. The initial exchange status is written through the handle.
-
-This ordering means incoming books can already be routed by CME ID while the
-Spark key is pending. Shared-memory consumers cannot discover the security yet.
-Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
-when `key() != 0`, so a trade received in this short window updates the cold
-last-trade state but is not added to the keyed trade log.
-
-### 4. Recorder tick: spreads, handles, then database
-
-Every 200 ms the recorder calls, in this exact order:
-
-1. `work_spreads()` tries to fill previously unresolved spread leg keys by
-   asking `CMESecurityIndex` for each leg exchange ID.
-2. `work_handles()` drains new handles. Outrights are converted to an API
-   security and queued for DB lookup immediately. A spread without all leg keys
-   is retained in `pending_handle` and its leg pointers are registered in
-   `unresolved_legs`.
-3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
-
-Because spread work precedes new-handle work, a newly seen unresolved spread
-normally waits until at least the next timer tick to resolve legs. It waits
-longer if any outright leg has not yet received its Spark key. A spread's own
-key is not requested until every leg key is non-zero.
-
-### 5. Lookup or generation
-
-`work_db()` first queries `(ns, key)`:
-
- Existing row: use its `rowid` as the key and invoke the callback.
- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
-  rowid inside a transaction, the row object receives that ID, and the callback
-  runs after the transaction completes.
-
-New inserts are throttled to roughly 200 per timer pass (the current `> 200`
-check can admit 201) to avoid delaying heartbeats during startup. The queues are
-TBB concurrent queues, but all SQLite access is serialized onto the recorder's
-dispatcher thread. `check_access()` asserts this single-thread rule after the
-first work pass records the dispatcher thread ID.
-
-### 6. Final assignment and visibility
-
-The queued callback captures a copy of `MDProducerHandle` and calls
-`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
-then increments `security_count`. From this point:
-
- shared-memory scans report the security;
- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
- trades and RFQs can be emitted with a non-zero key;
- resolved spreads can copy this key into their leg structures.
-
-### 7. Sunday preload is a fast path
-
-For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
-definitions from `securities.sqlite`. It converts them with
-`set_security_key=true`, so both the security key and spread leg keys come from
-the serialized API security. `MDProducer::add_security()` calls `set_key()`
-immediately, and no recorder lookup is required for that preloaded handle. The
-CME exchange ID is still restored into `numeric_id`, allowing the normal CME
-index to be built.
-
-## Generalized/newer pipeline in detail
-
-### 1. Topology and ownership
-
-`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
-`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
-`MarketDataGateway`, and starts the configured providers. For each `(venue,
-source)` returned by a provider, the gateway creates an `MdSink` containing:
-
- a venue-local `GeneralSecurityIndex`;
- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
- a `GeneralMarketDataListener` connecting the source to that producer/index.
-
-The source decides its callback thread. Some sources call the listener while
-starting; others own network or replay threads. The generalized listener calls
-the producer inline on that source thread. The process-wide `IDGenerator`
-serializes its own calls with a mutex, but the listener's transient
-`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
-expected to serialize security/event callbacks.
-
-| Generalized stage | Executing thread | DB call? |
-|---|---|---|
-| Build/parse domain definition | Source/adaptor thread | No |
-| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
-| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
-| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
-| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
-| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
-
-There are two important compatibility exceptions inside the provider framework:
-
- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
-  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
-  legacy flow documented above.
- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
-  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
-  producer/index. Its assignment is still synchronous and pre-populated, so it
-  resembles the newer ID flow even though it does not pass through
-  `GeneralMarketDataListener`.
-
-### 2. How a new ID is constructed
-
-For a normal live generalized adapter, ID creation happens **in the adapter,
-before** `GeneralMarketDataListener::on_securities()`:
-
-```text
-logical identity = (domain venue, domain security type, adapter-chosen name)
-Spark ID         = SQLite rowid for that identity in key_ids.sqlite
-```
-
-`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
-the enum prefixes:
-
-```text
-VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
-```
-
-It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
-row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
-next rowid. That integer is wrapped in `domain::SecurityID` and placed in
-`SecurityCommon::id`.
-
-The third identity component is adapter-specific, and this matters when trying
-to reproduce a mapping:
-
-| Source/adaptor family | Typical `name` sent to `IDGenerator` |
-|---|---|
-| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
-| Taifex | product/instrument ID string |
-| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
-| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
-| Emumba Deribit | extracted Emumba security descriptor |
-| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
-
-So “security name” alone is not a universal logical key. Reproduction requires
-the exact venue enum, exact security-type enum, and the exact adapter-specific
-name used at insertion time.
-
-### 3. What `GeneralMarketDataListener` does with that ID
-
-`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
-behavior is controlled by:
-
-```cpp
-const bool use_given_sec_key = true;
-```
-
-Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
-call the DB. It copies `security_common.id.value()` into its transient
-`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
-`_add_spot()` then calls `_copy_security_common()`, which sets:
-
-```cpp
-to.key = incoming_id;
-to.numeric_id = incoming_id;
-```
-
-This collapses the generalized source/correlation ID and Spark shared-memory
-key into the same number. The source's text `exchange_sec_id` remains on the
-domain security but is not copied into `FixedSecurity::numeric_id` by this
-path.
-
-`_commit_security()` then:
-
-1. calls `MDProducer::add_security()` with an already non-zero key;
-2. causes `MDProducerHandle::set_key()` to run immediately during that call;
-3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
-   number); and
-4. returns with the security already visible in shared memory.
-
-Because the sink's producer has no recorder, supplying `id == 0` would allocate
-the shared-memory entry but never assign or publish its key. Generalized live
-sources must therefore provide a valid non-zero ID before `on_securities()`.
-
-The listener contains an alternative branch for `use_given_sec_key == false`.
-That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
-sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
-also effectively unused. Turning the flag off is not a mechanical change: an
-adapter such as Databento allocates using numeric instrument ID while the
-listener branch would use `sec.desc()`, potentially creating a second key for
-the same instrument.
-
-There is also a spread failure edge case. `_find_sec(sec, true)` records the
-parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
-ID is absent, `_add_spread()` logs and returns without committing the spread. A
-later repeat of the parent definition is then classified as a duplicate and
-ignored, so definition ordering is a hard requirement in the current code.
-
-### 4. How later books, trades, and statistics find the security
-
-Every later domain event carries a `domain::SecurityID`:
-
- `Book::security_id()`;
- `TradeRecord::sec_id`; or
- `SecurityMarketDataStatistics::security_id`.
-
-The listener first uses `cc_sec_id_map` to translate that event ID to the value
-stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
-`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
-but it still serves as the admission check that the definition was seen.
-
-Most high-throughput adapters avoid SQLite on market-data events:
-
- Databento allocates on the definition and caches
-  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
- Taifex keeps the resolved ID in its security-details mapper.
- Other native subscriptions maintain comparable descriptor/ID maps.
-
-The Python bridge is a notable exception: `to_bts_security_()`,
-`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
-`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
-one SQLite `SELECT` for each converted Python definition, trade, or book. It
-returns the existing ID after the first insert, but it is still a DB call and
-takes the global `IDGenerator` mutex on every event.
-
-### 5. Persistence timing and latency
-
-The newer allocator is synchronous:
-
- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
-  thread.
- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
-  (SQLite autocommit), also on the caller's thread.
- Listener publication: no further DB call under the current
-  `use_given_sec_key` setting.
-
-The ID therefore exists in `key_ids.sqlite` before the adapter sends the
-definition to the listener. There is no intentional 200 ms publication delay,
-but the source callback/startup path directly pays DB and mutex latency. Bulk
-definition discovery performs one lookup and possibly one insert per security;
-`IDGenerator` does not batch those operations.
-
-WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
-not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
-occasional file-system work. The impact is usually a startup/definition cost
-when adapters cache the result, but it becomes hot-path latency for code such as
-the current Python bridge that resolves every event through the DB.
-
-`GeneralMarketDataListener` itself does not persist the full contract definition
-to `securities.sqlite`; that remains the separate security-cache/metadata
-persistence path described earlier.
-
-## Comparison: MDP3 recorder vs generalized allocator
-
-| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
-|---|---|---|
-| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
-| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
-| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
-| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
-| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
-| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
-| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
-| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
-| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
-| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
-| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
-| Batching | Yes for new rows | No |
-| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
-
-Neither algorithm generates a UUID or a content hash. Both ultimately use an
-opaque, database-assigned SQLite rowid. It is normally increasing and looks like
-a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
-used to *look up* the row is logical; the resulting number is not derivable from
-that tuple.
-
-## Can another system reconstruct the same keys?
-
-Not from `(venue, type, name)` alone. Another system can reconstruct the
-**logical lookup tuple**, but it cannot calculate the numeric Spark key because
-the key depends on the historical insertion order of every row in
-`key_ids.sqlite`.
-
-For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
-row and receive different IDs if other securities were inserted before it on
-one system. They are sequential opaque IDs, not deterministic hashes and not
-UUIDs.
-
-Ways to preserve cross-system identity are:
-
-1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
-   identical. This preserves existing numeric keys but requires coordinated
-   allocation for new ones.
-2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
-   import rows with their original rowids before either system allocates new
-   securities.
-3. Treat a stable external identity (venue + type + exchange instrument ID, or
-   another canonical instrument identifier) as the cross-system key and carry a
-   translation table to each system's local Spark key.
-4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
-   be computed independently, but collision handling, descriptor
-   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
-   with orders/recordings/databases all need explicit design.
-
-Copying only `securities.sqlite` is insufficient as an allocator strategy. It
-contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
-the mapping used to find or allocate those IDs. Conversely, copying only
-`key_ids.sqlite` preserves the numbers but not all contract metadata.
-
-## Other assignment paths in `sparkd`
-
-The two main pipelines reduce to four ways a producer handle can acquire a key:
-
-1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
-   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
-   Sunday preload, relay playback, and other replay paths use this mechanism.
-2. **Recorder-backed live input.** The input key is zero and the producer owns a
-   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
-   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
-   `numeric_id` and otherwise follows the same recorder/index design.
-3. **Service assignment.** `SyntheticSecurityService` calls
-   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
-   callback receives a copied `api::Security` with its top-level `id` set, then
-   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
-   rowid allocator.
-4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
-   never becomes public. This is not a valid production assignment path, but it
-   is the failure mode if a generalized source omits its ID.
-
-Relay and replay deserve care when reasoning about persistence:
-
- The generalized `MDReplayer` forwards the recorded domain security ID and
-  emits later books/statistics with the recorded `sec_key`, so the listener's
-  current use-given mode preserves the recording's numeric identity.
- The legacy relay producer converts a recorded `api::Security` with
-  `set_security_key=true`, so its shared-memory publication also preserves the
-  embedded key.
- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
-  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
-  separate mapping population step, not a deterministic derivation of the key.
-
-## Operational implications
-
- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
-  definition continuity matter.
- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
-  another host, even when all instruments are identical.
- Before moving a feed between adapters, compare the exact `name` each adapter
-  supplies to `IDGenerator`. Matching venue/type is insufficient.
- Cache generated IDs in the source after definition processing. Avoid calling
-  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
-  this guideline.
- Keep generated rowids below the 32-bit shared-memory ceiling until the
-  `security_key` ABI is widened or explicit bounds checking is added to the
-  generalized path.
- Ensure spread legs are announced before generalized spread parents.
- Treat `synchronous=0` as a durability decision. A copied DB should be taken
-  with SQLite-aware backup/checkpoint handling rather than by casually copying
-  active WAL files.
-
-Useful inspection queries (run against the configured DB path) are:
-
-```sql
-SELECT rowid AS spark_key, ns, key
-FROM store
-ORDER BY rowid;
-
-SELECT rowid AS spark_key, ns, key
-FROM store
-WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
-```
-
-## Source map
-
- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
-  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
-  and spread-leg resolution.
- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
-  synchronous generalized lookup/allocation and process-wide mutex.
- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
-  location, schema, WAL/synchronous pragmas, individual inserts, and batch
-  inserts.
- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
-  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
-  shared-memory allocation, recorder handoff, and ordered publication.
- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
-  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
-  widths and hot/cold storage fields.
- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
-  and
-  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
-  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
-  producer/index insertion order.
- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
-  current dedicated MDP3 dispatcher-thread ownership.
- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
-  and
-  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
-  generalized source/sink topology, incoming-ID mapping, fixed-security
-  conversion, and publication.
- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
-  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
-  and
-  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
-  representative adapter-side allocation and caching behavior.
- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
-  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
-  persistence of full security definitions.
- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
-  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
-  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
-  translation followed by Spark-keyed market-data consumption and MBO access.
- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
-  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
-  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
-  representative downstream use of Spark security keys.
- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
-  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
-  ID reuse.
@@ -8,7 +8,6 @@ RUN apt-get update && apt-get install -y ca-certificates
 RUN apt-get update && apt-get install -y libssh2-1-dev
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-get update && apt-get install -y sudo
-RUN apt-get update && apt-get install -y tzdata

 # GCC
 ARG gcc_major=13
@@ -19,11 +18,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
  && apt-get install -y dotnet8

-RUN apt-get update && apt-get install -y gdb
-RUN apt-get update && apt-get install -y fish
-RUN apt-get update && apt-get install -y vim
-RUN apt-get update && apt-get install -y less
-
 RUN locale-gen en_US.UTF-8

 ARG docker_user="bts"
@@ -1,11 +0,0 @@
-FROM ubuntu:24.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y \
-  fish \
-  less \
-  vim
-
-USER bts
-WORKDIR /home/bts
@@ -49,6 +49,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{00D21F82-61F4-467F-B125-268018A185E5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{513A3F0D-46B5-4AC2-B700-8671933F6347}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{D419D996-9870-4FF5-82AD-08370D1C945F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -275,6 +283,102 @@ Global
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.ActiveCfg = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.Build.0 = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.ActiveCfg = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.Build.0 = Release|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.ActiveCfg = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.Build.0 = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.Build.0 = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.ActiveCfg = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.Build.0 = Release|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.ActiveCfg = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.Build.0 = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.ActiveCfg = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.Build.0 = Release|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.ActiveCfg = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.Build.0 = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.Build.0 = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.ActiveCfg = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.Build.0 = Release|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -302,5 +406,9 @@ Global
 		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
 		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{00D21F82-61F4-467F-B125-268018A185E5} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{513A3F0D-46B5-4AC2-B700-8671933F6347} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{D419D996-9870-4FF5-82AD-08370D1C945F} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 	EndGlobalSection
 EndGlobal
@@ -102,10 +102,10 @@ namespace BCG
         List<ColumnInfo> col_info = new List<ColumnInfo>();
         col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
         foreach (Pair<string, Type> p in group_by_col_info)
-            col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
+            col_info.Add(new ColumnInfo(p.first ?? string.Empty, col_index++, ExceptionAssert.NotNull(p.second)));
         foreach (string tc_prefix in tc_prefixes)
            foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, ExceptionAssert.NotNull(p.second)));
         IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
         if (_schema == null)
            _schema = schema;
@@ -182,7 +182,7 @@ namespace BCG
         if (_transpsed_col_value_comparer_wrapper != null)
            tc_values.Sort(_transpsed_col_value_comparer_wrapper);
         List<string> prefixes = new List<string>();
-         foreach (Pair<int, object[]> p in tc_values)
+         foreach (Pair<int, object?[]> p in tc_values)
            prefixes.Add(tc_prefixes[p.first]);
         _set_schema(prefixes, group_by_col_info, table_name);
         BlueTable result = new BlueTable(_schema);
@@ -1,17 +1,17 @@
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
+using System.Diagnostics.CodeAnalysis;
 using System.Text;

-using BCG;
-using BCG.Utilities;
-
 namespace BCG
 {
   public class ConcatenateColumnsMapper : IStreamMapper
   {
      protected IDictionary myTempColMap = new SortedList ();
-      protected ArrayList myColMap;
-      protected BlueTableSchema mySchema;
+      protected ArrayList? myColMap;
+      protected BlueTableSchema? mySchema;

      public ConcatenateColumnsMapper(params string [] columnDefs)
      {
@@ -55,11 +55,7 @@ namespace BCG
            myColMap = new ArrayList (myTempColMap.Count);
            foreach (DictionaryEntry de in myTempColMap)
            {
-               string[] cols = de.Value as string[];
-               ArrayList temp = new ArrayList ();
-               foreach (string col in cols)
-                  temp.Add (inputSchema[col].colNo);
-               myColMap.Add (temp.ToArray(typeof(int)) as int[]);
+               myColMap.Add(Array.ConvertAll((string[])de.Value!, c => inputSchema[c].colNo));
            }
            myTempColMap.Clear ();
         }
@@ -72,6 +68,8 @@ namespace BCG

      public virtual IBlueRow map (IBlueRow row)
      {
+         requireSchema();
+         requireColMap();
         object[] retVal = new object[mySchema.columnInfo.Length];
         row.CopyTo (retVal, myColMap.Count);
         for (int colNo = 0; colNo < myColMap.Count; colNo ++)
@@ -90,11 +88,21 @@ namespace BCG
      /// <returns></returns>
      protected virtual object myGetConcatValue (IBlueRow row, int colNo)
      {
+         requireColMap();
         StringBuilder b = new StringBuilder();
-         foreach (int cn in myColMap[colNo] as int[])
-            b.Append(row[cn].ToString());
+         foreach (int cn in (int[])myColMap[colNo]!) {
+            b.Append(row[cn]?.ToString());
+         }
         return b.ToString ();
      }
+
+      [MemberNotNull(nameof(myColMap))]
+      protected void requireColMap() =>
+         ExceptionAssert.NotNull(myColMap);
+
+      [MemberNotNull(nameof(mySchema))]
+      void requireSchema() =>
+         ExceptionAssert.NotNull(mySchema);
   }

   public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper 
@@ -107,23 +115,21 @@ namespace BCG
         myDicts = new HashTableND [columnDefs.Length];
         int i = 0;
         foreach (DictionaryEntry de in myTempColMap)
-            myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
+            myDicts[i++] = new HashTableND(((string[])de.Value!).Length);
      }

      protected override object myGetConcatValue (IBlueRow row, int colNo)
      {
-         int[] cols = myColMap[colNo] as int[];
-         object[] vals = new object[cols.Length];
-         for (int i = 0; i < vals.Length; i++)
-            vals[i] = row[cols[i]];
+         requireColMap();
         HashTableND ht = myDicts[colNo];
-         object rv = ht.getValue (vals);
+         var vals = Array.ConvertAll((int[])myColMap[colNo]!, c => row[c]);
+         var rv = ht.getValue(vals);
         if (rv != null)
            return rv;
         else
         {
            rv = base.myGetConcatValue (row, colNo);
-            object[] args = new object[vals.Length + 1];
+            var args = new object?[vals.Length + 1];
            vals.CopyTo (args, 0);
            args[args.Length-1] = rv;
            ht.add (args);
@@ -14,8 +14,10 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
         : base (colMapExpression, null, columnNames) {}

-      public override IBlueRow? evaluate(IDataPacket packet) {
-         myOutputTable ??= new GroupByOutputTable(packet.Schema);
+
+      public override IBlueRow evaluate (DataPacket packet)
+      {
+         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
         string hashString = GroupByStream.getRepString (packet, myGroupByCols);
         if (packet.newRows.Count == 0)
            //if there are no rows for this group we remove the group from the output table
@@ -67,7 +67,7 @@ namespace BCG {
      }
      #endregion

-      public event FilterChangeDelegate onFilterChange;
+      public event FilterChangeDelegate? onFilterChange;

      #region IStreamFilter
      public bool filter(IBlueRow row) {
@@ -1,4 +1,4 @@
-using System;
+using Bts.DataStreamCore;

 namespace BCG
 {
@@ -10,8 +10,8 @@ namespace BCG
 	{
      protected string[] myGroupByCols;
      protected string myOutputColumn;
-      protected GroupByOutputTable myOutputTable;
-      protected string myColMapExpression;
+      protected GroupByOutputTable? myOutputTable;
+      protected string? myColMapExpression;

      #region constructors
 		public GroupByEvaluator (string outputCol, string[] groupByCols)
@@ -42,7 +42,7 @@ namespace BCG
         return retVal;
      }

-      public abstract IBlueRow? evaluate(IDataPacket dp);
+      public abstract IBlueRow evaluate (DataPacket dp) ;
      #endregion

      #region Helper methods
@@ -52,21 +52,20 @@ namespace BCG
      /// <param name="r"></param>
      /// <param name="id"></param>
      /// <returns></returns>
-      protected IBlueRow myProject (IBlueRow r, int id)
-      {
-         return r == null ? null :
-            myColMapExpression == null || myColMapExpression.Length < 1 ?
-               BlueDataUtility.getNewReplacedRow (r, id, "id") :
-               BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression),
-            id, "id");
-      }
-      
+      protected IBlueRow? myProject(IBlueRow r, int id) =>
+         r is null
+            ? null
+            : myColMapExpression is null || myColMapExpression.Length < 1
+               ? BlueDataUtility.getNewReplacedRow(r, id, "id")
+               : BlueDataUtility.getNewReplacedRow(myProjectRow(r, myColMapExpression), id, "id");
+
      /// <summary>
      /// Projects the id, conserving the id
      /// </summary>
      /// <param name="r"></param>
      /// <returns></returns>
-      protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
+      protected IBlueRow? myProject(IBlueRow r) =>
+         myProject(r, r is null ? int.MinValue : r.Id());
      #endregion

      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
@@ -1,5 +1,6 @@
+using Bts.DataStreamCore;
+
 using System;
-using BCG.Utilities;
 using System.Collections;

 namespace BCG
@@ -22,10 +23,12 @@ namespace BCG
      /// <param name="groupRepString"></param>
      public void addRow (IBlueRow row, string groupRepString)
      {
-         if (myDict[groupRepString] != null)
-            myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
+         var grp = myDict[groupRepString];
+         if (grp is not null) {
+            myTable.deleteRow((IBlueRow)grp);
+         }
         myTable.addRow (row);
-         myDict[groupRepString] = myTable[(int)row["id"]];
+         myDict[groupRepString] = myTable[row.Id()];
      }

      /// <summary>
@@ -35,7 +38,7 @@ namespace BCG
      public void removeRow (string groupRepString)
      {
         // BlueEventLog.logAssert (myDict[groupRepString] != null);
-         myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
+         myTable.deleteRow ((IBlueRow)myDict[groupRepString]!);
         myDict.Remove (groupRepString);
      }

@@ -48,11 +51,8 @@ namespace BCG
         myDict.Clear ();
      }

-      public IBlueRow getRow (string groupRepString)  
-      {
-         return myDict[groupRepString] == null ? null : 
-            myDict[groupRepString] as IBlueRow;
-      }
+      public IBlueRow? getRow(string groupRepString) =>
+         myDict[groupRepString] as IBlueRow;

      #region IBlueRowCollection members
      public IBlueRow this[int id]  {get{return myTable[id];}}
@@ -1,20 +1,21 @@
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
-using BCG.Utilities;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;

 namespace BCG {
   internal class GroupByStream : MethodDrivenStream {
      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
-         readonly Func<IDataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
+         private Func<DataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
            _groupby_cols = groupby_cols.ToArray();
            _groupby_func = groupby_func;
         }
-         public IBlueRow evaluate(IDataPacket packet) =>
-            _groupby_func(packet);
+         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }

         public string[] groupByColumns { get { return _groupby_cols; } }

@@ -26,14 +27,15 @@ namespace BCG {
      /// Serves as  a quick way to access all rows belonging to the same group
      /// </summary>
      private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable;
+      RowDictionaryTable? myBookKeepingTable;
      private IGroupByEvaluator myGroupByEvaluator;
-      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
+      // input schema, as output schema can differ (IGroupByEvaluator's
+      IBlueTableSchema? mySchema;
      //can map a schema to another
-      private IBlueTableSchema myOutputSchema;
+      IBlueTableSchema? myOutputSchema;
      private int myIdGenerator = 0;

-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }

      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -46,14 +48,14 @@ namespace BCG {
      }

      protected override void myHandleFirstCall() {
+         requireLastArrivedPacket();
         mySchema = myLastArrivedPacket.schema;
         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
            myGroupByEvaluator.groupByColumns);
         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
-               rowGroup, new BlueTable(mySchema)));
+            var newRow = myEvaluate(new DataPacket(rowGroup, rowGroup, new BlueTable(mySchema)));
            if (newRow != null)
               temp.addRow(newRow, myGetGroupString(rowGroup));
         }
@@ -79,15 +81,14 @@ namespace BCG {
         ArrayList handledAddGroups = new ArrayList();
         string groupRepString;

+         requireBookKeeping();
         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
            // BlueEventLog.logAssert (delGroup.Count > 0);
            groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) ?? new BlueTable(mySchema);
            handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
-            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) ?? new BlueTable(mySchema);
+            var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
            myFireSwitch |= myMergeRow(newRow, groupRepString);
         }

@@ -99,16 +100,16 @@ namespace BCG {
               //the loop above

               groupRepString = myGetGroupString(addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
+               var aggGroup = ExceptionAssert.NotNull(myBookKeepingTable.getGroup(groupRepString));
               // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
-                  new BlueTable(aggGroup.schema)));
+               var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, new BlueTable(aggGroup.schema)));
               myFireSwitch |= myMergeRow(newRow, groupRepString);
            }
         }
      }

-      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+      private bool myMergeRow(IBlueRow? newRow, string groupRepString) {
+         requireLastFiredPacket();
         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
         //BlueEventLog.logAssert (newRow != null || existingRow != null);
         if (existingRow == null && newRow == null)
@@ -116,6 +117,7 @@ namespace BCG {
         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
            return false;
         else if (existingRow == null) {
+            ArgumentNullException.ThrowIfNull(newRow);
            //row added for the first time
            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
@@ -138,6 +140,9 @@ namespace BCG {
      /// <param name="delGroups"></param>
      /// <param name="addGroups"></param>
      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
+         ExceptionAssert.NotNull(myBookKeepingTable);
+         requireLastFiredPacket();
+         requireLastArrivedPacket();
         delGroups =
            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
         addGroups =
@@ -151,16 +156,19 @@ namespace BCG {
      }

      protected override void myOnTableClear() {
-         if (myBookKeepingTable.Count == 0)
-            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
+         if (ExceptionAssert.NotNull(myBookKeepingTable).Count == 0) {
+            // ie there arent any rows, so no need to optimize the 'cleansing' process
+            return;  
+         }
+         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         ((GroupByOutputTable)myLastFiredPacket.newRows).clear();
         myBookKeepingTable.clear();
         myGroupByEvaluator.clear();
      }

-      private IBlueRow myEvaluate(DataPacket dp) {
+      private IBlueRow? myEvaluate(DataPacket dp) {
         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
         return evalRow == null ? null :
            myIdGuard ?
@@ -188,11 +196,11 @@ namespace BCG {
         }
      }

-      public static string getRepString(IDataPacket dp, string[] colNames) {
+      public static string getRepString(DataPacket dp, string[] colNames) {
         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
      }

-      public static IBlueRow getRepRow(IDataPacket dp) {
+      public static IBlueRow? getRepRow(DataPacket dp) {
         if (dp.newRows.Count > 0)
            return BlueDataUtility.extractRow(dp.newRows, 0);
         else if (dp.addedRows.Count > 0)
@@ -211,13 +219,13 @@ namespace BCG {
      /// <param name="newValue"></param>
      /// <param name="isAdd"></param>
      /// <returns></returns>
-      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+      public static object? incrementalAdd(object lastValue, object newValue, bool isAdd) {
         if (lastValue == null || lastValue.Equals(DBNull.Value))
            return newValue;
         // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
-         Object retVal = null;
+         object? retVal = null;
         if (newValue.GetType().Equals(typeof(int)))
            retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
         else if (newValue.GetType().Equals(typeof(Double)))
@@ -229,6 +237,9 @@ namespace BCG {
         return retVal;
      }

+      [MemberNotNull(nameof(myBookKeepingTable))]
+      void requireBookKeeping() =>
+         ExceptionAssert.NotNull(myBookKeepingTable);
   }

 }
@@ -1,3 +1,5 @@
+using System;
+
 namespace BCG
 {
   /// <summary>
@@ -12,7 +14,7 @@ namespace BCG
      /// deletetion) and the added and deletedRows being the increments. If a group has
      /// to be totally removed then a null should be returned
      /// </summary>
-      IBlueRow? evaluate(IDataPacket packet);
+      IBlueRow evaluate (DataPacket packet);

      /// <summary>
      /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator
@@ -14,8 +14,9 @@ namespace BCG {
        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
           : base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow? evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
@@ -1,10 +1,9 @@
-using System;
-using System.Collections;
-using BCG.Utilities;
-using System.IO;
-
 namespace BCG
 {
+   using Spark.Core.Diagnostics;
+
+   using System.Diagnostics.CodeAnalysis;
+
   /// <summary>
   /// Some basic event handling functionality factorred out of any Stream whose behaviour
   /// is driven by some interface passed on by the user
@@ -12,8 +11,8 @@ namespace BCG
   public abstract class MethodDrivenStream : ViewStream
   {
      protected ViewStream myStream;
-      protected DataPacket myLastArrivedPacket;
-      protected DataPacket myLastFiredPacket;
+      protected DataPacket? myLastArrivedPacket;
+      protected DataPacket? myLastFiredPacket;
      protected bool myFireSwitch = true;

      public MethodDrivenStream(ViewStream vs)  {myStream = vs;}
@@ -59,6 +58,7 @@ namespace BCG
      /// To handle the first call, the LastFiredPacket is supposed to be initialized and filled
      /// with data.
      /// </summary>
+      [MemberNotNull(nameof(myLastFiredPacket))]
      protected abstract void myHandleFirstCall ();

      /// <summary>
@@ -89,5 +89,13 @@ namespace BCG
      {
         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
      }
+
+      [MemberNotNull(nameof(myLastArrivedPacket))]
+      protected void requireLastArrivedPacket() =>
+         ExceptionAssert.NotNull(myLastArrivedPacket);
+
+      [MemberNotNull(nameof(myLastFiredPacket))]
+      protected void requireLastFiredPacket() =>
+         ExceptionAssert.NotNull(myLastFiredPacket);
   }
 }
@@ -1,3 +1,7 @@
+using System;
+
+using BCG.Utilities;
+
 namespace BCG {
    public class MinGroupByEvaluator : MaxGroupByEvaluator {
        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -6,8 +10,8 @@ namespace BCG {
        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
           base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow? evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
@@ -1,3 +1,5 @@
+using System;
+
 namespace BCG {
    /// <summary>
    /// Computes the sum for a given column
@@ -8,8 +10,8 @@ namespace BCG {
        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
           : base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;

            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
            return sumRow;
@@ -31,7 +33,7 @@ namespace BCG {

        #region IGroupByEvaluator Members

-        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }

        public string[] groupByColumns { get { return myGroupByCols; } }

@@ -49,8 +51,9 @@ namespace BCG {
        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
           : base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow evaluate(IDataPacket dp) {
-            myOutputTable ??= new GroupByOutputTable(dp.Schema);
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
            return sumRow;
        }
@@ -65,7 +68,7 @@ namespace BCG {

        #region IGroupByEvaluator Members

-        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }

        public string[] groupByColumns { get { return myGroupByCols; } }

@@ -1,5 +1,7 @@
 using mbts;

+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections.Generic;

@@ -35,7 +37,7 @@ namespace BCG {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
         foreach (Triplet<ViewStream, string, string> r in join_specs) {
            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
+            string[] visible_cols = ExceptionAssert.NotNull(r.third).Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
@@ -47,7 +49,7 @@ namespace BCG {
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add(new(ExceptionAssert.NotNull(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
         return rv.ToArray();
      }
@@ -290,6 +290,7 @@ namespace BCG
               }
            }
         }
+         ExceptionAssert.NotNull(myState);
      }

      private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
@@ -1,3 +1,5 @@
+using Spark.Core.Diagnostics;
+
 using System;

 namespace BCG
@@ -18,16 +20,21 @@ namespace BCG

      public override void onTableClear (bool isLeft)
      {
+         requireSchema();
+         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
         myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
         myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft)
+         if (!isLeft) {
+            ExceptionAssert.NotNull(myLeftDataPacket);
            foreach (IBlueRow row in myLeftDataPacket.newRows)
               ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
+         }
      }

      protected override void myInitFiredPacket ()
      {
+         requireSchema();
         myLastFiredPacket = new DataPacket ();
         myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
         myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -15,7 +15,7 @@ namespace BCG
         : this (r, null, null)
      {}

-      public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
+      public SimpleNode(IBlueRow? r, SimpleNode? nx, SimpleNode? pv) {
         row = r;
         next = nx;
         prev = pv;
@@ -1,5 +1,7 @@
 using bts;

+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections.Generic;

@@ -60,7 +62,7 @@ namespace BCG {
            if (columnAndType.Length == 1)
               ci[n] = new ColumnInfo(c, n, typeof(double));
            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+               ci[n] = new(columnAndType[0], n, ExceptionAssert.NotNull(Type.GetType("System." + columnAndType[1], true)));
            n++;
         }

@@ -2,8 +2,6 @@ using BCG;

 using MongoDB.Driver.Core.WireProtocol.Messages;

-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -39,7 +37,7 @@ namespace bts {
      }

      void _handle_stream_update(object dp_obj) {
-         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
+         DataPacket dp = dp_obj as DataPacket;
         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
         :
@@ -23,6 +23,6 @@ namespace BCG
      /// fires an event when the mapper 
      /// changes itself
      /// </summary>
-      event MapperChangeDelegate onMapperChange;
+      event MapperChangeDelegate? onMapperChange;
   }
 }
@@ -75,6 +75,7 @@ namespace BCG {

      protected override void myHandleFirstCall() {
         ArgumentNullException.ThrowIfNull(myMapper);
+         requireLastArrivedPacket();
         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
         myLastFiredPacket = new DataPacket();
         myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -85,10 +86,12 @@ namespace BCG {
      }

      protected override void myHandleIncrementalAdd() {
+         requireLastArrivedPacket();
         if (myLastArrivedPacket.addedRows.Count == 0) {
            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
            return;
         }
+         requireLastFiredPacket();
         ArgumentNullException.ThrowIfNull(myMapper);
         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
@@ -101,10 +104,12 @@ namespace BCG {
      }

      protected override void myHandleIncrementalDelete() {
+         requireLastArrivedPacket();
         if (myLastArrivedPacket.deletedRows.Count == 0) {
            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
            return;
         }
+         requireLastFiredPacket();
         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
@@ -127,6 +132,7 @@ namespace BCG {
      }

      protected override void myOnTableClear() {
+         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
@@ -47,14 +47,14 @@ namespace BCG {
            _map_rows = map_rows;
         }

+         event MapperChangeDelegate? ISuperStreamMapper.onMapperChange { add { } remove { } }
+
         public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }

         public IBlueRow[] map(IBlueRow r) {
            IEnumerable<IBlueRow> rows = _map_rows(r);
            return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
         }
-
-         public event MapperChangeDelegate? onMapperChange;
      }

      //maps the id of the underlying stream to the
@@ -608,8 +608,8 @@ namespace BCG {
            // 5. deserialize the state into a stream op
            str_io_msg_p = deser.deserialize(content_p);
            // 6. check to make sure the state's rows are good
-            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
-                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
+            passes &= str_io_msg_p.op == StreamIOMessage.StreamOp.CREATE
+               && BlueDataUtility.compute_equality(cr_rows, ExceptionAssert.NotNull(str_io_msg_p.create_message).rows);
            return new TestResult {
                test_result = passes,
                test_name = "run_pubsub_server_stream_tag_handler_test"
@@ -692,7 +692,7 @@ namespace BCG {
            var content = st_data.get_current_content();
            var msg_p = deser.deserialize(content);
            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
-            return msg_p.create_message.rows;
+            return ExceptionAssert.NotNull(msg_p.create_message).rows;

        }

@@ -1,5 +1,7 @@
 using Bts.ColumnSortExpressions;

+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
         var otherVal = otherRow[col.Name];
         int rv = myIsNull(this[col.Name]) ?
            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
+            (myIsNull(otherVal) ? 1 : ExceptionAssert.Argument.Is<IComparable>(this[col.Name]).CompareTo(otherVal));
         rv *= col.IsAscending ? 1 : -1;
         return rv;
      }
@@ -1,4 +1,6 @@
-using System;
+using Spark.Core.Diagnostics;
+
+using System;
 using System.Collections.Generic;
 using System.IO;

@@ -60,13 +62,13 @@ namespace BCG
            List<string> columnInfoStrParts = new List<string>(columnInfoStr.Split(new char[] { ':' }, StringSplitOptions.None));
            if (columnInfoStrParts.Count != 2)
            {
-               throw (new Exception(String.Format("Unable to decode schema string '{0}'", schemaStr)));
+               throw new Exception($"Unable to decode schema string '{schemaStr}'");
            }

-            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, Type.GetType(columnInfoStrParts[1])));
+            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, ExceptionAssert.NotNull(Type.GetType(columnInfoStrParts[1]))));
         }

-         return (new BlueTableSchema(columnInfos.ToArray(), ""));
+         return new BlueTableSchema(columnInfos.ToArray(), string.Empty);
      }

      private IBlueRow myGetRow(IBlueTableSchema schema, string rowStr)
@@ -75,7 +77,7 @@ namespace BCG

         if (schema.columnInfo.Length != valueStrs.Count)
         {
-            throw (new Exception(String.Format("Unable to decode row string '{0}'", rowStr)));
+            throw new Exception($"Unable to decode row string '{rowStr}'");
         }

         List<object> values = new List<object>();
@@ -84,7 +86,7 @@ namespace BCG
            values.Add(Convert.ChangeType(valueStrs[i], schema.columnInfo[i].colType));
         }

-         return (new BlueRow(schema, values.ToArray()));
+         return new BlueRow(schema, values.ToArray());
      }
      #endregion

@@ -1,6 +1,7 @@
 using BCG.Utilities.General;

 using System;
+using System.Collections.Generic;
 using System.Threading;

 namespace BCG
@@ -97,7 +98,7 @@ namespace BCG
         // BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread, 
         //    "onTargetDoWork should only be called from target thread");
         var updates = _queued_updates.dequeueAll();
-         foreach (var u in updates) {
+         foreach (var u in updates.ExceptNull()) {
            if (u.subscription == _current_target_subscription)
               myFireEvent(u.dp);
         }
@@ -1,7 +1,9 @@
-using System;
-
 using BCG.Utilities.General;

+using Spark.Core.Diagnostics;
+
+using System;
+
 namespace BCG {
   public abstract class TimeUnitStream : ViewStream, IWorker {
      protected TimeSpan myOffset;
@@ -46,8 +48,8 @@ namespace BCG {
            myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));


-         if (myGetLatestRow() == null ||
-            !myGetLatestRow()["time"].Equals(dt)) {
+         var latestRow = myGetLatestRow();
+         if (latestRow is null || !ExceptionAssert.Argument.Is<DateTime>(latestRow["time"]).Equals(dt)) {
            IBlueRow ar = new BlueRow(myCollection.schema,
               myId++, dt);
            IBlueRow dr = myCollection.addRowWithReturn(ar);
@@ -71,11 +73,10 @@ namespace BCG {

      #region Private Helper methods

-      private IBlueRow myGetLatestRow() {
-         return myCollection.Count == 0 ?
-            null :
-            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
-      }
+      IBlueRow? myGetLatestRow() =>
+         myCollection.Count == 0
+            ? null
+            : BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);

      private IBlueRowCollection myConvertToCollection(IBlueRow r) {
         BlueTable retVal = new BlueTable(myCollection.schema);
@@ -1,43 +1,42 @@
-using System;
+namespace BCG;

-namespace BCG {
-    public class TopStream : MethodDrivenStream {
-        private int myTopCount;
+public class TopStream : MethodDrivenStream {
+   int myTopCount;

-        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
+   public TopStream(ViewStream stream, int topCount)
+      : base(stream) =>
+      myTopCount = topCount;

-        public TopStream(ViewStream stream,
-           int topCount,
-           string sortExpression)
-           :
-           this(sortExpression == null ?
-           stream :
-           new SortStream(stream, sortExpression), topCount) { }
+   public TopStream(ViewStream stream, int topCount, string sortExpression)
+      : this(sortExpression is null ? stream : new SortStream(stream, sortExpression), topCount) { }

-        protected override void myOnTableClear() {
-            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-        }
+   protected override void myOnTableClear() {
+      requireLastFiredPacket();
+      myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+      myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+   }

-        protected override void myHandleFirstCall() {
-            myLastFiredPacket = new DataPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
-        }
+   protected override void myHandleFirstCall() {
+      requireLastArrivedPacket();
+      myLastFiredPacket = new DataPacket();
+      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+      myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+   }

-        protected override void myHandleIncrementalAdd() { }
+   protected override void myHandleIncrementalAdd() { }

-        protected override void myHandleIncrementalDelete() {
-            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-               oldRows,
-               myLastFiredPacket.schema.columnNames,
-               out myLastFiredPacket.deletedRows,
-               out myLastFiredPacket.addedRows);
-            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-        }
-    }
+   protected override void myHandleIncrementalDelete() {
+      requireLastArrivedPacket();
+      requireLastFiredPacket();
+      IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+      BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+         oldRows,
+         myLastFiredPacket.schema.columnNames,
+         out myLastFiredPacket.deletedRows,
+         out myLastFiredPacket.addedRows);
+      myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+   }
 }
@@ -1,12 +1,10 @@
-
-using System;
 using System.Collections;

 namespace BCG {

   public class AlphabetizeColumnsStream : ViewStream {
      private ViewStream mySrc;
-      private IBlueTableSchema mySchema = null;
+      IBlueTableSchema? mySchema = null;

      public AlphabetizeColumnsStream(ViewStream src) {
         mySrc = src;
@@ -1,12 +1,12 @@
-using System;
+using System.Linq;

 namespace BCG
 {
 	public class AppendStreamModified : ViewStream
 	{
      private AppendCollection myColl = new AppendCollection ();
-      private ViewStream myLeftStream, myRightstream;
-	
+      ViewStream? myLeftStream, myRightstream;
+
      public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
 		{
         myLeftStream = leftStream;
@@ -15,11 +15,24 @@ namespace BCG

      public AppendStreamModified (ViewStream[] vss)
      {
+         switch (vss.Length) {
+            case 0:
+               break;
+            case 1:
+               mySetRealSource (vss[0]);
+               break;
+            default:
+               mySetRealSource(
+                  vss[1..].Aggregate(
+                     new AppendStreamModified(new[] { vss[0] }),
+                     (asm, vs) => new AppendStreamModified(asm, vs)));
+               break;
+         }
         if (vss.Length == 1)
            mySetRealSource (vss[0]);
         else
         {
-            AppendStreamModified rv = null;
+            AppendStreamModified? rv = null;
            foreach (ViewStream s in vss)
               rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s); 
            mySetRealSource (rv);
@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;

-using global::Spark.Core.Disposables;
-
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;

+using Spark.Core.Disposables;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -8,7 +8,6 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;

-using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;

@@ -43,6 +43,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "..\cs\Bts.Collections\Bts.Collections.csproj", "{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "..\cs\Bts.Core\Bts.Core.csproj", "{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "..\cs\Bts.Disposables\Bts.Disposables.csproj", "{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "..\cs\Bts.Numerics\Bts.Numerics.csproj", "{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -293,6 +301,54 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.Build.0 = Debug|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.Build.0 = Debug|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.ActiveCfg = Release|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.Build.0 = Release|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.ActiveCfg = Release|Any CPU
+		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.Build.0 = Release|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.Build.0 = Debug|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.Build.0 = Release|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.ActiveCfg = Release|Any CPU
+		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.Build.0 = Release|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.Build.0 = Debug|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.Build.0 = Debug|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.ActiveCfg = Release|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.Build.0 = Release|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.ActiveCfg = Release|Any CPU
+		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.Build.0 = Release|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.Build.0 = Debug|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.Build.0 = Debug|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.ActiveCfg = Release|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.Build.0 = Release|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.ActiveCfg = Release|Any CPU
+		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -5,7 +5,7 @@ using BCG.Utilities.General;

 using Bts.DataStreamCore;

-using global::Spark.Core.Collections;
+using Spark.Core.Collections;

 public class TestDriverStream : ViewStream, IWorker {
   readonly IBlueTableSchema schema;
@@ -22,7 +22,6 @@ namespace BCG {
   /// with PubSub, as some clients are still using ManagedSubcribers)
   /// </summary>
   public class ManagedSubscriber : IManagedSubscriber, IWorker, IDisposable {
-      private static ManagedSubscriber? _self;
      private ISubscriberCore _subcore;
      // private IDictionary myTypedDispatchTable = new Hashtable(),
      //    myRawDispatchTable = new Hashtable();
@@ -34,7 +33,6 @@ namespace BCG {
      ManagedSubscriber(ISubscriberCore subsCore) { _subcore = subsCore; }

      public void doWork() {
-         var msg_map = new Dictionary<string, List<bts.PubSubServerMessage>>();
         var updates = _subcore.get_updates(TimeSpan.Zero);
         if (updates.Count() == 0)
            return;
@@ -45,11 +45,7 @@ namespace BCG
         get { return myConnInfo; }
      }

-      override public string ToString()
-      {
-         Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
-         return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
-      }
-
+      public override string ToString() =>
+         $"type: {myMsgType} conn: {myConnInfo} msg: {myMsg?.Length ?? 0}";
   }
 }
@@ -46,7 +46,7 @@ namespace bts {
            System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
            _accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { });
            if (ex is OutOfMemoryException)
-               throw ex;
+               throw;
            return;
         }
         new_sock.LingerState = new LingerOption(false, 0);
@@ -1,9 +1,8 @@
+using bts;
+
 using System;
 using System.IO;
-using System.Collections;
-using bts;
-using System.Threading;
-using System.Reflection;
+using System.Text;

 namespace BCG {
   /// <summary>
@@ -69,7 +68,8 @@ namespace BCG {

      public static byte[] getBytesFromStream(params Stream[] streams) {
         long totalSize = 0;
-         BinaryWriter bw = new BinaryWriter(new MemoryStream());
+         using var ms = new MemoryStream();
+         using var bw = new BinaryWriter(ms, Encoding.UTF8, true);
         foreach (Stream s in streams)
            totalSize += s.Length - s.Position;
         bw.Write(Convert.ToInt32(totalSize));
@@ -78,7 +78,8 @@ namespace BCG {
            int lengthToReadForThisStream = Convert.ToInt32(s.Length - s.Position);
            bw.Write(br.ReadBytes(lengthToReadForThisStream));
         }
-         return (bw.BaseStream as MemoryStream).ToArray();
+         bw.Flush();
+         return ms.ToArray();
      }

      public static byte[] read_bytes(Stream str) {
@@ -92,7 +92,7 @@ namespace BCG {

      public void publish(PubSubClientMessage msg) {
         Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
-         Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
+         Log.assert(!string.IsNullOrEmpty(msg.pub_msg?.tag_type));
         bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
         _main_queue.enqueue(msg);
      }
@@ -143,7 +143,7 @@ namespace BCG {
      public void doWork() {
         if (_connection_cb != null) {
            var conns = _conn_callback_queue.dequeueAll();
-            foreach (var ci in conns) {
+            foreach (var ci in conns.ExceptNull()) {
               _connection_cb(ci.client_url, ci.status);
            }
         }
@@ -175,7 +175,7 @@ namespace BCG {
            case RemoteMessageType.Data:
               // if callback is registered fire it right away, else enqueue it
               if (_data_cb != null)
-                  _data_cb(sender_rci.ToString(), message.message);
+                  _data_cb(sender_rci.ToString(), message.message ?? Array.Empty<byte>());
               else
                  _received_data_queue.enqueue(new Pair<string, byte[]>(sender_rci.ToString(), message.message));
               break;
@@ -258,7 +258,7 @@ namespace BCG {
      // (which gets called in another thread)
      private void _handle_new_connections() {
         var new_conns = _new_connections.dequeueAll();
-         foreach (ITCPConnection conn in new_conns) {
+         foreach (var conn in new_conns.ExceptNull()) {
            var peer_rci = conn.get_peer_address();
            bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections conn: {conn}");
            ///to stop it from being garbabge collected immediatly
@@ -16,8 +16,8 @@ public interface IPubSubState {

   ICollection<string> all_tags { get; }

-   byte[] get_content(string tag);
-   string getTagType(string tag);
+   byte[]? get_content(string tag);
+   string? getTagType(string tag);
   void get_all_connections(
      string tag,
      out Dictionary<string, string> subscribers,
@@ -188,11 +188,11 @@ namespace BCG {
         _state.handle_sub(sub_msg);   // state keeps the list of all subscribers
         ///to all these new subscriber s, send the existing data
         foreach (string tag in sub_msg.tags.empty_if_null()) {
-            byte[] data = _state.get_content(tag);
+            byte[]? data = _state.get_content(tag);
            //ie data has been published once
            if (data != null) {
               bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
-               string tag_type_ = _state.getTagType(tag);
+               string? tag_type_ = _state.getTagType(tag);
               var msg_to_send = new PubSubServerMessage() {
                  tag = tag,
                  tag_type = tag_type_,
@@ -148,7 +148,7 @@ namespace BCG {
            return new string[] { };
      }

-      public byte[] get_content(string tag) {
+      public byte[]? get_content(string tag) {
         if (!_tag_to_data_map.ContainsKey(tag))
            return null;
         var rv = _tag_to_data_map[tag].get_current_content();
@@ -157,7 +157,7 @@ namespace BCG {
         return rv;
      }

-      public string getTagType(string tag) {
+      public string? getTagType(string tag) {
         if (!_tag_to_data_map.ContainsKey(tag))
            return null;
         else
@@ -20,7 +20,7 @@ namespace bts {
   public class SingleTagData {
      private readonly IPubSubServerTagHandler? _tag_handler;

-      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
+      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler? tag_handler) {
         bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
         this.tag = tag;
         this.tag_type = tag_type;
@@ -46,7 +46,6 @@ namespace BCG {
      }

      public void onConnection(ITCPConnection connection) {
-         string ip; int port;
         var peer_rci = connection.get_peer_address();
         bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop");
         if (_listener != null) {
@@ -5,9 +5,10 @@ using BCG;
 using bts.api;
 using bts.md;

+using Bts.DataStream;
 using Bts.DataStreamCore;

-using global::Spark.Api.MarketData;
+using Spark.Api.MarketData;

 public class BookSchemaBuilder {
   public int NumLevels { get; set; } = 5;
@@ -2,7 +2,7 @@ namespace Bts.SharedStreams;

 using bts.api;

-using global::Spark.Api.Orders;
+using Spark.Api.Orders;

 public static class WellKnownColumns {
   public static (string Name, Type Type) AskNotional => (WellKnownColumnNames.AskNotional, typeof(double));
@@ -10,7 +10,7 @@
   [DataStreamUserRespresentation("nl")]
   public class BlueOrbNLStream : ViewStream {
      private readonly string _nl_prompt;
-      private ViewStream _nl_to_dsl_stream, _dsl_stream;
+      private ViewStream? _nl_to_dsl_stream, _dsl_stream;
      private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
      public BlueOrbNLStream(string nl_prompt) {
         _nl_prompt = nl_prompt;
@@ -85,7 +85,8 @@ namespace bts.BlueOrb {
                .GetProperty("choices")[0]
                .GetProperty("message")
                .GetProperty("content")
-                .GetString();
+                .GetString()
+                ?? string.Empty;

            //return gpt_response;
            DateTime t1 = DateTime.Now;
@@ -4,6 +4,8 @@ using BCG.Utilities.General;
 using Bts.PubSub;
 using Bts.SharedStreams.Tables;

+using Spark.Core.Diagnostics;
+
 namespace bts {
   // TODO: Use dependency injection to instantiate this stream, and others like it that depend on
   // ISubscriberCore(Factory). Until then, hide it from DSQA.
@@ -40,7 +42,7 @@ namespace bts {
         foreach (var u in updates) {
            if (u.tag != "System_AllStatistics")
               continue;
-            var tag_infos = TagStatsInfo.deserialize(u.data);
+            var tag_infos = TagStatsInfo.deserialize(ExceptionAssert.NotNull(u.data));
            foreach (var ti in tag_infos) {
               _tag_map[ti.tag] = ti;
            }
@@ -187,24 +187,4 @@
      Name="Test"
      DependsOnTargets="Build"></Target>

-  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
-    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
-    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
-  </Target>
-
-  <ItemGroup>
-    <Content Include="@(NativeWindowsAsset)"
-             Condition="Exists('@(NativeWindowsAsset)')"
-             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
-             CopyToOutputDirectory="PreserveNewest"
-             Pack="true"
-             PackagePath="runtimes/win-x64/native/" />
-    <Content Include="@(NativeLinuxAsset)"
-             Condition="Exists('@(NativeLinuxAsset)')"
-             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
-             CopyToOutputDirectory="PreserveNewest"
-             Pack="true"
-             PackagePath="runtimes/linux-x64/native/" />
-  </ItemGroup>
-
 </Project>
@@ -2,14 +2,12 @@ ARG IMAGE="sparkd"
 ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
 ARG SOURCE_DIR="install/sparkd-release-gcc"
 ARG PYTHON_REQUIREMENTS="requirements.txt"
-ARG SPARKD_ENTRYPOINT="src/app_sparkd/sparkd-entrypoint.sh"

 # sparkd_base
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_base
 ARG CONFIG_DIR
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS
-ARG SPARKD_ENTRYPOINT

 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -23,7 +21,6 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
 COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/.local/share/bts/sparkd/conf"
-COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"

 # sparkd
@@ -38,7 +35,6 @@ COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/.local/share/bts/sparkd/da
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_symbols
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS="requirements.txt"
-ARG SPARKD_ENTRYPOINT

 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -51,7 +47,6 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"

 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
-COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"

 # selected image
@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/all_crypto
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/gateio_dev/requirements.txt
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
@@ -1,11 +0,0 @@
-{
-  "currencies": [
-    "USDT"
-  ],
-  "futures": [
-    "BTC",
-    "ETH",
-    "SOL",
-    "XRP"
-  ]
-}
@@ -1,16 +0,0 @@
-{
-  "currencies": [
-    "USDT"
-  ],
-  "futures": [
-    "BTC",
-    "ETH",
-    "SOL",
-    "XRP"
-  ],
-  "use_testnet": false,
-  "logging": {
-    "console_level": "WARNING",
-    "file_level": "INFO"
-  }
-}
@@ -1,71 +0,0 @@
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    # python_md_providers {
-    #   venue: VENUE_GATEIO
-    #   module_path: "python_md_providers"
-    #   provider_type: "gateio_md_provider.GateIoMdProvider"
-    # }
-    python_md_providers {
-      venue: VENUE_BYBIT
-      module_path: "python_interop_plugins"
-      provider_type: "bybit_md_provider.BybitMdProvider"
-    }
-    binance_usds_futures {
-      num_depth_levels: 5
-      url_rest_base: "https://fapi.binance.com"
-      url_ws_base: "wss://fstream.binance.com"
-      security_criteria {
-        clauses {
-          security_types: SECURITY_TYPE_FUTURE
-          symbols: @SymbolLists.list[Gateio].symbol
-        }
-      }
-    }
-     okx {
-       num_depth_levels: 5
-       url_rest_base: "https://www.okx.com"
-       url_ws_base: "wss://ws.okx.com:8443"
-       security_criteria {
-         clauses {
-           security_types: SECURITY_TYPE_FUTURE
-           symbols: @SymbolLists.list[Gateio].symbol
-         }
-       }
-       is_application_ping_pong_enabled: true
-       application_ping_interval_ms: 20000
-       application_pong_timeout_ms: 10000
-       is_protocol_ping_pong_enabled: true
-       protocol_ping_interval_ms: 60000
-       protocol_pong_timeout_ms: 30000
-     }
-  }
-}
@@ -1,32 +0,0 @@
-  SymbolLists {
-  value {
-    list {
-      key: "Gateio"
-      symbol: "SOL"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "XRP"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GATEIO"
-      spot: @SymbolLists.list[Gateio].symbol
-    }
-    venues {
-      name: "VENUE_BYBIT"
-      future: @SymbolLists.list[Gateio].symbol
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: @SymbolLists.list[Gateio].symbol
-    }
-    venues {
-      name: "VENUE_OKX"
-      future: @SymbolLists.list[Gateio].symbol
-    }
-  }
-}
@@ -1,7 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_dev_replay_aws
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_dev_replay_aws/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
-SPARKD__SHM_SIZE=512m
-SPARKD__MEMORY=1g
@@ -1,61 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    # list {
-    #   key: "GlbxOptions"
-    #   symbol: "E1A"
-    #   symbol: "E1B"
-    #   symbol: "E1C"
-    #   symbol: "E1D"
-    #   symbol: "E2A"
-    #   symbol: "E2B"
-    #   symbol: "E2C"
-    #   symbol: "E2D"
-    #   symbol: "E3A"
-    #   symbol: "E3B"
-    #   symbol: "E3C"
-    #   symbol: "E3D"
-    #   symbol: "E4A"
-    #   symbol: "E4B"
-    #   symbol: "E4C"
-    #   symbol: "E4D"
-    #   symbol: "E5A"
-    #   symbol: "E5B"
-    #   symbol: "E5C"
-    #   symbol: "E5D"
-    #   symbol: "EW1"
-    #   symbol: "EW2"
-    #   symbol: "EW3"
-    #   symbol: "EW4"
-    #   symbol: "ESH9"
-    #   symbol: "ESZ6"
-    #   symbol: "ESH0"
-    #   symbol: "ESM6"
-    #   symbol: "ESZ7"
-    #   symbol: "ESH6"
-    #   symbol: "ESM7"
-    #   symbol: "ESZ8"
-    #   symbol: "ESH7"
-    #   symbol: "ESH8"
-    #   symbol: "ESU6"
-    #   symbol: "ESZ5"
-    #   symbol: "ESZ9"
-    #   symbol: "E1D"
-    #   symbol: "ESZ5"
-    #   symbol: "ES"
-    # }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        # option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}
@@ -1,10 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
-SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
-SPARKD__SHM_SIZE=8g
-SPARKD__MEMORY=16g
-SPARKD__CONFIG_PORT=9000
-SPARKD__QUICKBUS_PORT=9001
-SPARKD__SIM_FIX_PORT=9002
@@ -1,5 +0,0 @@
-numpy==2.2.6
-pybit==5.11.0
-requests==2.31.0
-websocket-client==1.8.0
-websockets==15.0.1
@@ -1,140 +0,0 @@
-DeploymentType {
-  value {
-    type: "dev"
-  }
-}
-PublisherOptions {
-   value {
-      network_interface_a: 'ens65f1'
-      network_interface_b: 'ens65f0'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
-}
-
-JetFIX {
-  value {
-    connection {
-      name: "maw_004_64"
-      venue: VENUE_GLBX
-      socket_connect_host_ip: "69.50.112.177"
-      socket_connect_port: 44040
-      sender_compid: "MAW004"
-      target_compid: "CME"
-      target_subid: "64"
-      cme_market_segment_id: 64
-      start_time: "00:10:00"
-      end_time: "23:55:00"
-      start_day: "sunday"
-      end_day: "friday"
-      seconds_between_hbs: 15
-      sender_subid: "BTS"
-      seconds_to_pause: 2
-      max_connection_attempts: 100
-      use_ilink3: true
-    }
-  }
-}
-OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-            poll: false 
-      hosted_actor: "*"
-    }
-    fallback_cust_order_handling_inst: "Y"
-  }
-}
-NotSimExecution {
-  value {
-    entries {
-      venue: VENUE_GLBX
-    }
-  }
-}
-Ports {
-   value {
-      config_port: 9000
-      qbus_port: 9001
-      glbx_sim_fix_port: 9002
-      ice_sim_fix_port: 9003
-      replicator_port: 9080
-   }
-}
@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
+SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
@@ -24,9 +24,38 @@ PublisherOptions {
  }
 }

+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Warn"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+      subsystem {
+        name: "Http"
+        level: "Diag"
+      }
+    }
+  }
+}
+
 MarketDataConfig {
  value {
-   replay {
+    replay {
      venue: VENUE_GLBX
    }
  }
@@ -0,0 +1,8 @@
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GLBX"
+      future: "ES"
+    }
+  }
+}
@@ -1,7 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
-SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf
-SPARKD__SHM_SIZE=8g
-SPARKD__MEMORY=16g
@@ -1,5 +0,0 @@
-numpy==2.2.6
-pybit==5.11.0
-requests==2.31.0
-websocket-client==1.8.0
-websockets==15.0.1
@@ -1,152 +0,0 @@
-DeploymentType {
-  value {
-    type: "prod"
-  }
-}
-PublisherOptions {
-   value {
-      network_interface_a: 'enp193s0f0np0'
-      network_interface_b: 'enp193s0f1np1'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
-}
-
-JetFIX {
-  value {
-    connection {
-      name: "u6p_7qb_54"
-      venue: VENUE_GLBX
-      socket_connect_host_ip: "205.209.201.97"
-      socket_connect_port: 10979
-      sender_compid: "U6P7QB"
-      target_compid: "CME"
-      target_subid: "54"
-      cme_market_segment_id: 54
-      start_time: "00:10:00"
-      end_time: "23:55:00"
-      start_day: "sunday"
-      end_day: "friday"
-      seconds_between_hbs: 15
-      sender_subid: "BTS"
-      seconds_to_pause: 1
-      max_order_msgs_per_window: 300
-      max_connection_attempts: 2
-      use_ilink3: true
-    }
-   connection {
-      name: "u6p_7qb_64"
-      venue: VENUE_GLBX
-      socket_connect_host_ip: "205.209.201.1"
-      socket_connect_port: 10979
-      sender_compid: "U6P7QB"
-      target_compid: "CME"
-      target_subid: "64"
-      cme_market_segment_id: 64
-      start_time: "00:10:00"
-      end_time: "23:55:00"
-      start_day: "sunday"
-      end_day: "friday"
-      seconds_between_hbs: 15
-      sender_subid: "BTS"
-      seconds_to_pause: 1
-      max_connection_attempts: 2
-      max_order_msgs_per_window: 300
-      use_ilink3: true
-    }
-  }
-}
-OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-            poll: false 
-      hosted_actor: "*"
-    }
-    fallback_cust_order_handling_inst: "Y"
-  }
-}
-NotSimExecution {
-  value {
-    entries {
-      venue: VENUE_GLBX
-    }
-  }
-}
@@ -1,61 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    list {
-      key: "GlbxOptions"
-      symbol: "E1A"
-      symbol: "E1B"
-      symbol: "E1C"
-      symbol: "E1D"
-      symbol: "E2A"
-      symbol: "E2B"
-      symbol: "E2C"
-      symbol: "E2D"
-      symbol: "E3A"
-      symbol: "E3B"
-      symbol: "E3C"
-      symbol: "E3D"
-      symbol: "E4A"
-      symbol: "E4B"
-      symbol: "E4C"
-      symbol: "E4D"
-      symbol: "E5A"
-      symbol: "E5B"
-      symbol: "E5C"
-      symbol: "E5D"
-      symbol: "EW1"
-      symbol: "EW2"
-      symbol: "EW3"
-      symbol: "EW4"
-      symbol: "ESH9"
-      symbol: "ESZ6"
-      symbol: "ESH0"
-      symbol: "ESM6"
-      symbol: "ESZ7"
-      symbol: "ESH6"
-      symbol: "ESM7"
-      symbol: "ESZ8"
-      symbol: "ESH7"
-      symbol: "ESH8"
-      symbol: "ESU6"
-      symbol: "ESZ5"
-      symbol: "ESZ9"
-      symbol: "E1D"
-      symbol: "ESZ5"
-      symbol: "ES"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}
@@ -1,7 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
-SPARKD__DATA_DIR=/home/dk/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/dk/8000/sparkdata/conf
-SPARKD__SHM_SIZE=8g
-SPARKD__MEMORY=16g
@@ -1,5 +0,0 @@
-numpy==2.2.6
-pybit==5.11.0
-requests==2.31.0
-websocket-client==1.8.0
-websockets==15.0.1
@@ -1,101 +0,0 @@
-DeploymentType {
-  value {
-    type: "prod"
-  }
-}
-PublisherOptions {
-   value {
-      network_interface_a: 'enp193s0f0np0'
-      network_interface_b: 'enp193s0f1np1'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
-}
-
-OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-            poll: false 
-      hosted_actor: "*"
-    }
-    fallback_cust_order_handling_inst: "Y"
-  }
-}
@@ -1,61 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    list {
-      key: "GlbxOptions"
-      symbol: "E1A"
-      symbol: "E1B"
-      symbol: "E1C"
-      symbol: "E1D"
-      symbol: "E2A"
-      symbol: "E2B"
-      symbol: "E2C"
-      symbol: "E2D"
-      symbol: "E3A"
-      symbol: "E3B"
-      symbol: "E3C"
-      symbol: "E3D"
-      symbol: "E4A"
-      symbol: "E4B"
-      symbol: "E4C"
-      symbol: "E4D"
-      symbol: "E5A"
-      symbol: "E5B"
-      symbol: "E5C"
-      symbol: "E5D"
-      symbol: "EW1"
-      symbol: "EW2"
-      symbol: "EW3"
-      symbol: "EW4"
-      symbol: "ESH9"
-      symbol: "ESZ6"
-      symbol: "ESH0"
-      symbol: "ESM6"
-      symbol: "ESZ7"
-      symbol: "ESH6"
-      symbol: "ESM7"
-      symbol: "ESZ8"
-      symbol: "ESH7"
-      symbol: "ESH8"
-      symbol: "ESU6"
-      symbol: "ESZ5"
-      symbol: "ESZ9"
-      symbol: "E1D"
-      symbol: "ESZ5"
-      symbol: "ES"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}
@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
+SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf
@@ -1,12 +1,18 @@
-DeploymentType {
+JetFIX {
  value {
-    type: "prod"
+    connection {
+      name: "glbx_1"
+      venue: VENUE_GLBX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
  }
 }
+
 PublisherOptions {
   value {
-      network_interface_a: 'ens5'
-      network_interface_b: 'ens5'
+      network_interface_a: 'ens3f0np0'
+      network_interface_b: 'ens3f1np1'
      #these get overriden if not entered here
      heartbeat_ms: 5000
      heartbeat_boot_seconds: 60
@@ -35,7 +41,7 @@ CMEOptions {
      use_tcp_replay: false
      recovery_queue_size: 640000
      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
      #max_gap_recovery_cache_size: 4
      use_natural_refresh: false
      join_snapshot_on_gap: false
@@ -67,17 +73,32 @@ MarketDataConfig {
       unique_check_value: 15
       use_databento: false
    }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
+  }
 }
+
+# JetFIX {
+#    value {
+#     #connection: @JetFIXSimConnection.sim
+#     connection {
+#     name: "mlx_004_54"
+#     venue: VENUE_GLBX
+#     socket_connect_host_ip: "69.50.112.191"
+#     socket_connect_port: 42876
+#     sender_compid: "MLX004"
+#     target_compid: "CME"
+#     target_subid: "54"
+#     cme_market_segment_id: 54
+#     start_time: "00:10:00"
+#     end_time: "23:55:00"
+#     start_day: "sunday"
+#     end_day: "friday"
+#     seconds_between_hbs: 30
+#     sender_subid: "BTS"
+#     use_ilink3: true
+#   #   raw_data: "S4PMA"
+#   #   auto_fix_seq_num: true
+#   }
+#    }
+# }
+
+
@@ -1,4 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay/requirements.txt
-TZ=America/Chicago
-SPARKD__MD_RECORDING_DIR=/home/wedbushprod/8000/md_recording
@@ -1 +0,0 @@
-requests==2.31.0
@@ -1,46 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    list {
-      key: "GlbxOptions"
-      symbol: "E1A"
-      symbol: "E1B"
-      symbol: "E1C"
-      symbol: "E1D"
-      symbol: "E2A"
-      symbol: "E2B"
-      symbol: "E2C"
-      symbol: "E2D"
-      symbol: "E3A"
-      symbol: "E3B"
-      symbol: "E3C"
-      symbol: "E3D"
-      symbol: "E4A"
-      symbol: "E4B"
-      symbol: "E4C"
-      symbol: "E4D"
-      symbol: "E5A"
-      symbol: "E5B"
-      symbol: "E5C"
-      symbol: "E5D"
-      symbol: "ES"
-      symbol: "EW1"
-      symbol: "EW2"
-      symbol: "EW3"
-      symbol: "EW4"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}
@@ -1,9 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay_wb
-SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay_wb/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
-SPARKD__MD_RECORDING_DIR=/home/wedbushprod/md_recording
-SPARKD__CONFIG_PORT=9000
-SPARKD__QUICKBUS_PORT=9001
-SPARKD__SIM_FIX_PORT=9002
@@ -1,5 +0,0 @@
-numpy==2.2.6
-pybit==5.11.0
-requests==2.31.0
-websocket-client==1.8.0
-websockets==15.0.1
@@ -1,46 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-Ports {
-   value {
-      config_port: 9000
-      qbus_port: 9001
-      glbx_sim_fix_port: 9002
-      ice_sim_fix_port: 9003
-   }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-     replay {
-      venue: VENUE_GLBX
-    }
-  }
-}
-
@@ -1,61 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    list {
-      key: "GlbxOptions"
-      symbol: "E1A"
-      symbol: "E1B"
-      symbol: "E1C"
-      symbol: "E1D"
-      symbol: "E2A"
-      symbol: "E2B"
-      symbol: "E2C"
-      symbol: "E2D"
-      symbol: "E3A"
-      symbol: "E3B"
-      symbol: "E3C"
-      symbol: "E3D"
-      symbol: "E4A"
-      symbol: "E4B"
-      symbol: "E4C"
-      symbol: "E4D"
-      symbol: "E5A"
-      symbol: "E5B"
-      symbol: "E5C"
-      symbol: "E5D"
-      symbol: "EW1"
-      symbol: "EW2"
-      symbol: "EW3"
-      symbol: "EW4"
-      symbol: "ESH9"
-      symbol: "ESZ6"
-      symbol: "ESH0"
-      symbol: "ESM6"
-      symbol: "ESZ7"
-      symbol: "ESH6"
-      symbol: "ESM7"
-      symbol: "ESZ8"
-      symbol: "ESH7"
-      symbol: "ESH8"
-      symbol: "ESU6"
-      symbol: "ESZ5"
-      symbol: "ESZ9"
-      symbol: "E1D"
-      symbol: "ESZ5"
-      symbol: "ES"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-init_timezone() {
-  tz="${1:-$TZ}"
-  zoneinfo_dir=/usr/share/zoneinfo
-  if [ -f "$zoneinfo_dir/$tz" -a "$(readlink -f /etc/localtime)" != "$zoneinfo_dir/$tz" ]; then
-    echo "Setting locales and timezone to $tz"
-    if [ -e /etc/localtime ]; then
-      sudo -n rm -f /etc/localtime
-    fi
-    sudo -n ln -snf "$zoneinfo_dir/$tz" /etc/localtime
-    echo $tz | sudo -n tee /etc/timezone > /dev/null
-  fi
-}
-
-load_env() {
-  f="$1"
-  if [ -e "$f" ]; then
-    echo "Sourcing $f"
-    set -a
-    . "$f"
-    set +a
-  fi
-}
-
-symlink() {
-  src="$1"
-  dst="$2"
-  if [ -L "$dst" ]; then
-    rm "$dst"
-  fi
-  ln -s "$src" "$dst"
-}
-
-roll_log() {
-  f="$1"
-  if [ -e "$f" ]; then
-    mv "$f" "${f}_$(date '+%Y%m%d-%H%M')"
-  fi
-}
-
-# Acquire sudo permission once for the script (needed for a few calls)
-echo bts | sudo -S -v
-data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
-sudo -n chown -R bts:bts "$data_root"
-load_env "$data_root/conf/.env.local"
-init_timezone "$TZ"
-roll_log "$data_root/data/logging.sparkd/log"
-symlink "$data_root/conf/plugins" /home/bts/bin/plugins
-
-if  [ -e "/md_recording" -a ! -e "/home/bts/.local/share/bts/sparkd/data/md_recording" ]; then
-  symlink "/md_recording" /home/bts/.local/share/bts/sparkd/data/md_recording
-fi
-
-bash -c "${@:-./sparkd run}"
@@ -17,6 +17,7 @@
 #include "spark/orderman/OrderServiceServer.h"
 #include "spark/orderman/events/TradeSummarySignalDispatcher.h"
 #include "spark/orderman/order_factory/DynamicOrderRegistry.h"
+#include "spark/os/os.h"
 #include "spark/os/spark_data_directory.h"
 #include "spark/persistence/PersistenceServiceServer.h"
 #include "spark/plugin/SparkServices.h"
@@ -31,7 +32,6 @@
 #include "sparkd/execution/execution_gateway_provider.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
-#include "sparkd/md/market_data_session.h"
 #include "sparkd/md/recording/MDRecorder.h"
 #include "sparkd/position/MarginServiceServer.h"
 #include "sparkd/position/PositionServiceServer.h"
@@ -46,7 +46,6 @@
 namespace bts::cmds {

 using bts::api::ServiceContext;
-using bts::md::MarketDataSession;
 using bts::md::MDClient;
 using bts::md::TradeSummarySignalDispatcher;
 using bts::metrics::MetricsContainer;
@@ -108,7 +107,7 @@ static void execute() {
   auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
   // start work loop in background
   py_plugin_man->start_work_thread();
-   MarketDataSession md_session{py_plugin_man};
+   md::cmds::Run::run(py_plugin_man);

   api::synthetic::SyntheticSecurityService synthetic_service(sc);
   bts::md::MDClient mdclient(
@@ -1,11 +1,11 @@
 #include "DumpSecurity.h"
 #include "spark/api/domain/SecurityType.h"
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/core/TableFormatter.h"
 #include "spark/domain_model/Security.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/domain_model/SecurityType.h"
 #include "spark/domain_model/Venue.h"
-#include "spark/format/table.h"
 #include "spark/md/Diag/ShmDumper.h"
 #include "spark/md/SecurityDB.h"
 #include "spark/md/SymbolLoader.h"
@@ -23,8 +23,7 @@ using bts::api::SecurityType;
 using bts::api::SecurityType_Name;
 using bts::api::Venue;
 using bts::api::venue_parse_smart;
-using bts::format::format_table;
-using bts::format::detail::TableSchema;
+using bts::core::TableFormatter;
 using bts::md::securitytype_of;
 using bts::md::SymbolLoader;
 using bts::md::db::load_metadata;
@@ -70,7 +69,7 @@ void print_securities(function<bool(const Security &)> predicate) {

 void print_db_securities(function<bool(const Security &)> predicate) {
   vector<Security> secs;
-   const TableSchema<Security> schema{
+   const TableFormatter::Schema<Security> schema{
       make_tuple(
           "id", 12, [](const Security &s) { return std::to_string(s.id()); }),
       make_tuple(
@@ -94,7 +93,7 @@ void print_db_securities(function<bool(const Security &)> predicate) {
      secs.push_back(sec);
   };
   load_securities([](auto) { return true; }, on_sec);
-   format_table(std::cout, secs, schema);
+   TableFormatter::format(std::cout, secs, schema);
 }

 template <class T, class ParseT = function<bool(const string &, T *)>>
@@ -109,8 +109,7 @@ struct ExecutionGatewayProvider::PImpl {
         return make_gw<ICEExecutionGateway>(conn, conn, *l, o);
      case api::VENUE_GLBX:
         if (conn.use_ilink3()) {
-            return make_gw<ilink3::GLBXiLink3ExecutionGateway>(
-                conn, conn, *l, o);
+            make_gw<ilink3::GLBXiLink3ExecutionGateway>(conn, conn, *l, o);
         }
         return make_gw<GLBXExecutionGateway>(conn, conn, *l, o);
      case api::VENUE_DERIBIT:
@@ -23,7 +23,7 @@
 #include "spark/api/services/native_order_type.h"
 #include "spark/api/services/time_in_force.h"
 #include "spark/config/protos/jetfix.pb.h"
-#include "spark/core/time_stamp.h"
+#include "spark/core/TimeStamp.h"
 #include "spark/core/start_stop_state_machine.h"
 #include "spark/domain/tick_utils.h"
 #include "spark/domain_model/Security.h"
@@ -4,7 +4,7 @@
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/core/FMT.h"
-#include "spark/core/time_span.h"
+#include "spark/core/TimeSpan.h"
 #include "spark/domain/DomainSecurityUtils.h"
 #include "spark/domain/SecurityID.h"
 #include "spark/domain_model/Venue.h"
@@ -141,6 +141,5 @@ MarketDataGateway::MarketDataGateway(
 MarketDataGateway::~MarketDataGateway() = default;

 void MarketDataGateway::start() { me_->start(); }
-void MarketDataGateway::stop() { me_->stop(); }

 } // namespace bts::md
Author	SHA1	Message	Date
John Colvin	0c0d2ac911	refactor: Add new glbx variable ticks to security (shm and proto)	2026-03-04 08:00:40 +00:00
John Colvin	c55d6447f4	diag: Add logs to MDP3Listener::onOption	2026-03-03 15:28:13 +00:00
John Colvin	48c2641069	refactor: Enable nullability in Bts.Spark.Api	2026-03-03 15:06:06 +00:00
John Colvin	303e4be09e	build: Upgrade cs LangVersion 11 -> 12	2026-03-03 15:06:06 +00:00
John Colvin	45ff7fea95	refactor: Rename SparkApiCredentials -> SparkCredentials	2026-03-03 15:06:06 +00:00
John Colvin	6462a6a56c	refactor: Rename SparkdLoginService -> SparkLoginService This makes it consistent with other type names (we don't use "Sparkd" anywhere else).	2026-03-03 15:06:06 +00:00
John Colvin	f842a0528f	refactor: Rename SparkdCredentials.cs -> SparkApiCredentials.cs Matches the class name	2026-03-03 15:06:06 +00:00
John Colvin	74c9023eac	cleanup: Remove unnecessary subscription wrapper in cs sdk	2026-03-03 15:06:06 +00:00
John Colvin	c7edd8403f	cleanup: Fix indentation in TopStream Whitespace changes	2026-03-03 15:06:06 +00:00
John Colvin	0a8e066c86	cleanup: Fix build warnings in EqualityComparer Nullability issues	2026-03-03 15:06:06 +00:00
John Colvin	2db4d23ca9	refactor: Rename Optional members PascalCase Also reformat code according to new style guidlines	2026-03-03 15:06:06 +00:00
John Colvin	f876061236	refactor: Replace ToDictionaryCrushed/Dupes w/ DuplicateKeyPolicy enum Overload ToDictionary with this enum instead.	2026-03-03 15:06:06 +00:00
John Colvin	40482d51e4	refactor: Replace find_first_index with FindIndex	2026-03-03 15:06:06 +00:00
John Colvin	28e899fd10	cleanup: Fix cs build warnings	2026-03-03 15:06:06 +00:00
John Colvin	2f7e75fec0	refactor: Break up Bts.Spark.Core into a few diff projs This is in anticipation of decoupling BlueOrb from Spark. * Bts.Collections * Bts.Core * Bts.Disposables * Bts.Numerics	2026-03-03 15:06:06 +00:00
John Colvin	749fed15ad	build: Clean up spark_client.sln * Remove unused csprojs related to old sdk * Remove erroneous build type config	2026-03-03 15:06:06 +00:00