added more detail missing security log in gmdl

config: new config for ophirs new sim server
Add: simple doc to expplain security handling in sparkd
2026-10-26 03:32:49 +00:00 · 2026-10-21 13:09:14 +00:00 · 2026-09-04 13:28:03 +00:00 · 2026-09-04 13:27:12 +00:00 · 2026-09-04 13:23:25 +00:00 · 2026-09-01 05:28:30 +00:00
651 changed files with 25372 additions and 3687 deletions
@@ -29,6 +29,9 @@
 #     tools/deploy.sh --help
 #

+### Host/Runtime ###
+TZ=Etc/UTC
+
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -38,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
+# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
+# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
+# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -45,6 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh

 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
@@ -183,11 +183,10 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
      "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -214,7 +213,7 @@
      "type": "coreclr",
      "request": "launch",
      "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
+      "preLaunchTask": "Build Spark Sdk (Debug)",
      "windows": {
        "program": "Bts.Spark.Sdk.Demo.exe",
      },
@@ -286,7 +285,7 @@
      "windows": {
        "program": "Spark.exe",
        "args": [
-          "--sparkd",
+          "--Spark:ConfigServer",
          "${input:sparkConfigServer}"
        ],
        "cwd": "build/Debug/bin/net8.0-windows",
@@ -449,14 +448,15 @@
      "default": "Local",
    },
    {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
      "type": "pickString",
-      "description": "PubSub Configuration",
+      "description": "PubSubServer Configuration",
      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
      ],
-      "default": "PubSubServer",
+      "default": "Development",
    },
    {
      "id": "sdkDemoEnvironment",
@@ -17,14 +17,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{3BF4DF96-5AFF-4674-866A-71A8090627FF}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{62067659-AA2E-4CB5-B7CE-0B38EAA92080}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -95,54 +87,6 @@ Global
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.Build.0 = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.Build.0 = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.ActiveCfg = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.Build.0 = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.ActiveCfg = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.Build.0 = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.Build.0 = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.Build.0 = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.ActiveCfg = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.Build.0 = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.Build.0 = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.Build.0 = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.Build.0 = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.ActiveCfg = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.Build.0 = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.ActiveCfg = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.Build.0 = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.Build.0 = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.Build.0 = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.ActiveCfg = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -154,9 +98,5 @@ Global
 		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal
@@ -51,14 +51,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{802ECF0B-7264-4168-95D1-158E47FB65CC}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{0354655D-2394-42BB-B1D4-2D75B9BCABF1}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -579,102 +571,6 @@ Global
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.ActiveCfg = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.Build.0 = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.ActiveCfg = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.Build.0 = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.ActiveCfg = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.Build.0 = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.ActiveCfg = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.Build.0 = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.ActiveCfg = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.Build.0 = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.Build.0 = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.ActiveCfg = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.Build.0 = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.ActiveCfg = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.Build.0 = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.ActiveCfg = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.Build.0 = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -696,10 +592,6 @@ Global
 		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{802ECF0B-7264-4168-95D1-158E47FB65CC} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
@@ -2,6 +2,7 @@ services:
  sparkd:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
@@ -9,24 +10,35 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-shell:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+    build:
+      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: "do sleep 1; done"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
@@ -42,6 +54,7 @@ services:
  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
@@ -50,26 +63,8 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
@@ -77,6 +72,7 @@ services:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
@@ -84,7 +80,65 @@ services:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
-
+  sparkd-multicast:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
+  
+  sparkd-multicast-cert:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
  sparkd-tail:
    image: bluetradingsystems/linux_dotnet8
    volumes:
@@ -195,3 +249,6 @@ volumes:
    name: "md_publisher_22019"
  historic_md:
    name: "historic_md"
+  md_recording:
+    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
+    external: true
@@ -0,0 +1,942 @@
+# Security ID handling in `sparkd`
+
+This document traces how security identifiers are created, translated, persisted,
+indexed, and published by the `sparkd` market-data paths. It focuses on the
+legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
+
+## Executive summary
+
+`sparkd` uses two identifiers that must not be confused:
+
+| Identifier | Meaning | Typical field | Lifetime |
+|---|---|---|---|
+| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
+| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
+
+`invalid_security_key` is `0`; a security with key `0` is not yet publicly
+discoverable through the shared-memory consumer path.
+
+The two principal paths differ:
+
+- **MDP3/OnixS:** the definition callback sets only the CME ID in
+  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
+  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
+  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
+  calls `MDProducerHandle::set_key()`.
+- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
+  currently uses the ID already present on the incoming domain security
+  (`use_given_sec_key == true`). Live adapters normally obtain that ID
+  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
+  it to both `FixedSecurity::key` and `numeric_id`, so
+  `MDProducer::add_security()` publishes it synchronously. Only the listener's
+  own fallback `IDGenerator` call is bypassed by the current constant.
+
+## End-to-end diagrams
+
+### CME MDP3 / OnixS
+
+```mermaid
+sequenceDiagram
+    participant FE as OnixS feed-engine thread
+    participant L as MDP3Listener
+    participant P as MDProducer / shared memory
+    participant Q as SecurityIDRecorder queues
+    participant D as sparkd dispatcher thread
+    participant DB as key_ids.sqlite
+
+    FE->>L: instrument definition (CME SecurityID)
+    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
+    L->>P: add_security(FixedSecurity)
+    P->>P: allocate hot + cold entries with key 0
+    P->>Q: ensure_key(handle)
+    L->>L: index[CME SecurityID] = handle
+    Note over FE,L: Books can route by CME ID while the Spark key is pending
+    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
+    Q->>DB: find (namespace, descriptor), or insert a new row
+    DB-->>Q: SQLite rowid = Spark security key
+    Q->>P: handle.set_key(rowid)
+    P->>P: hot.key first, then cold security.key, then increment security_count
+    Note over P: The security is now public to shared-memory consumers
+```
+
+### Generalized/newer pipeline (current behavior)
+
+```mermaid
+sequenceDiagram
+    participant A as Venue adapter / plugin
+    participant DB as IDGenerator / key_ids.sqlite
+    participant G as GeneralMarketDataListener
+    participant I as GeneralSecurityIndex
+    participant P as MDProducer / shared memory
+
+    A->>DB: get_or_add(venue, type, adapter key)
+    DB-->>A: SQLite rowid as domain SecurityID
+    A->>G: on_securities(domain Security list with ID)
+    G->>G: _find_sec() - use incoming security_common.id
+    G->>G: _copy_security_common() - key = id, numeric_id = id
+    G->>P: add_security(FixedSecurity with non-zero key)
+    P->>P: set_key() during add_security()
+    G->>I: index[id] = handle
+    Note over G,P: No asynchronous ID assignment in the current configuration
+```
+
+## Data-flow charts
+
+The sequence diagrams above emphasize timing and thread transitions. These
+flowcharts emphasize which identifier is carried through each representation
+and where data is retained.
+
+### MDP3 / legacy data flow
+
+```mermaid
+flowchart TD
+    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
+
+    ONIXS --> DEF["MDP3Listener definition callback"]
+    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
+
+    FIXED --> PRODUCER["MDProducer::add_security"]
+    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
+    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
+
+    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
+
+    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
+    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
+    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
+    UNRESOLVED --> CMEINDEX
+    CMEINDEX --> LEGS
+    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
+
+    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
+    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
+    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
+    INSERT --> ROWID
+
+    ROWID --> SETKEY["MDProducerHandle::set_key"]
+    SETKEY --> HOT["Write hot-entry key"]
+    HOT --> COLD["Write cold security key"]
+    COLD --> COUNT["Increment security_count"]
+    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
+
+    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
+    UPDATE --> CMEINDEX
+    CMEINDEX --> HANDLE["MDProducerHandle"]
+    HANDLE --> LIVE["Update live shared-memory data"]
+    PUBLIC --> LIVE
+```
+
+The CME index is usable before the Spark key is assigned, so feed updates can
+reach the internal handle while the security is still hidden from consumers.
+The persistent rowid crosses the publication boundary only at `set_key()`.
+
+### Generalized/newer data flow
+
+```mermaid
+flowchart TD
+    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
+
+    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
+    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
+    IDINPUT --> MUTEX["IDGenerator global mutex"]
+    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
+    SELECT -- Existing row --> ROWID["SQLite rowid"]
+    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
+    INSERT --> ROWID
+    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
+    IDTYPE -- Recorded or pre-keyed --> DOMAINID
+
+    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
+    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
+    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
+    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
+
+    CONVERT --> PRODUCER["MDProducer::add_security"]
+    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
+    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
+    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
+
+    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
+    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
+    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
+    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
+    PYHOT --> EVENTID
+    PYHOT -. uses same mutex and SQLite path .-> MUTEX
+    EVENTID --> CCMAP
+    CCMAP --> GENERALINDEX
+    GENERALINDEX --> HANDLE["MDProducerHandle"]
+    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
+    SHM --> SHMUPDATE
+
+```
+
+In the normal newer live path, persistence occurs before the definition enters
+the generalized listener. The DB call therefore adds synchronous adapter-side
+latency, but publication needs no later timer. Most adapters use an in-memory
+map for subsequent events; the dotted Python path is the documented exception.
+
+## The publication boundary
+
+`MDProducerHandle::set_key()` is the definitive publication point for the
+legacy shared-memory representation. Its write ordering is intentional:
+
+1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
+2. Write the key into the cold security entry.
+3. Increment the cold header's public `security_count`.
+
+The cold security key is written last because consumers treat a non-zero cold
+key as visibility. At that moment the corresponding hot entry must already be
+addressable. `set_key()` only permits the transition from `0` to one key; it is
+not a key-update mechanism.
+
+## Identifier representations
+
+The names are historically inconsistent. In particular, many protobuf fields
+named `security_id` contain the **Spark key**, not the exchange's ID.
+
+| Layer | Exchange/feed identifier | Spark identifier |
+|---|---|---|
+| MDP3 wire/OnixS | `securityId()` | not present |
+| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
+| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
+| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
+| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
+| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
+| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
+| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
+
+The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
+shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
+64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
+allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
+no explicit checked conversion.
+
+## Which parts use exchange IDs, descriptors, or Spark security IDs?
+
+The three identifier classes serve different jobs:
+
+- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
+- **Descriptor/natural key:** recognizes a definition logically at the
+  persistence, discovery, or human-facing boundary.
+- **Spark security ID/key:** joins the instrument to books, orders, fills,
+  positions, and services after publication.
+
+The core downstream system does not repeatedly identify a security from its
+description. The description is primarily used to obtain a Spark key; after
+that boundary, most joins use the numeric Spark key.
+
+### Component-level usage matrix
+
+| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
+|---|---|---|---|
+| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
+| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
+| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
+| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
+| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
+| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
+| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
+| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
+| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
+| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
+| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
+| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
+| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
+| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
+| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
+| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
+| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
+| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
+| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
+| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
+
+### Exchange/external IDs: where they stop
+
+In MDP3 the separation is explicit:
+
+```text
+CME SecurityID
+    -> FixedSecurity.numeric_id
+    -> CMESecurityIndex lookup
+    -> MDProducerHandle
+    -> Spark security_key for downstream publication
+```
+
+The external CME ID continues to route feed messages but does not become the
+order/position key. This permits the same downstream Spark key to survive a
+restart as long as the logical descriptor mapping remains in
+`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
+
+The generalized path collapses more of these roles:
+
+```text
+exchange/vendor identity
+    -> adapter calls IDGenerator
+    -> domain::SecurityID
+    -> FixedSecurity.key and FixedSecurity.numeric_id
+    -> GeneralSecurityIndex and downstream Spark key
+```
+
+`SecurityCommon::exchange_sec_id` may still contain the original textual
+external identifier, but `GeneralMarketDataListener` does not use it to route
+events and does not put it in `numeric_id`. The adapter must translate later
+events to `domain::SecurityID` before invoking the listener.
+
+### Descriptor/name users
+
+Descriptors are used in three main places:
+
+1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
+   descriptor, while `IDGenerator` uses its caller-provided `name`.
+2. **Definition lookup and discovery:** `SecurityCache` supports
+   venue/type/name lookup; symbol filters and administrative tools use security
+   metadata to select or display instruments.
+3. **Diagnostics:** logs include symbols and descriptions so humans can
+   understand a numeric key.
+
+Descriptions generally do not drive the high-rate book, trade, order, fill, or
+position paths. Those paths use numeric IDs after definition resolution.
+
+It is important that “descriptor” does not mean one universal canonical field:
+
+- Legacy CME uses Globex `security_desc` from tag 55.
+- Many generalized adapters use `sec.desc()` or an exchange instrument name.
+- Databento uses its numeric instrument ID converted to a string.
+- Python uses `exchange_security_id`.
+
+Therefore, two adapters may describe the same economic contract but request
+different logical rows and receive different Spark IDs.
+
+### Spark security-ID users
+
+Once a security is published, its Spark key is the dominant system identifier:
+
+```mermaid
+flowchart LR
+    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
+    KEY --> MDC["MDConsumer and book subscriptions"]
+    KEY --> MBO["MBO snapshots and queries"]
+    KEY --> OMS["Orders, algorithms, and spread legs"]
+    KEY --> FILLS["Fills and execution records"]
+    KEY --> POS["Positions, PNL, and risk"]
+    KEY --> API["Services and client APIs"]
+    KEY --> REC["Relay and market-data recordings"]
+    KEY --> SECDB["Serialized security definitions"]
+```
+
+Fields named `security_id`, `security_key`, or top-level `Security.id()` in
+these downstream areas almost always mean the Spark key. They should not be
+compared directly with CME `SecurityID`, Databento `instrument_id`, or another
+venue/vendor identifier without going through the relevant adapter/index
+mapping.
+
+## Persistence and allocation
+
+### The authoritative key mapping: `key_ids.sqlite`
+
+Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
+file is:
+
+```text
+<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
+```
+
+The database contains one table and two indexes:
+
+```sql
+CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
+CREATE UNIQUE INDEX key_index ON store(ns, key);
+CREATE INDEX ns_index ON store(ns);
+```
+
+The SQLite `rowid` is the generated Spark key. `ns` is normally
+`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
+descriptor chosen by the caller. `value` is normally empty. Therefore ID
+stability depends on all three of these remaining stable:
+
+- the configured `data_path` and preservation of this database;
+- the venue/type namespace;
+- the descriptor supplied by the feed adapter.
+
+The two allocator implementations share the file and convention:
+
+- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
+  uses `securities::descriptor()`. For CME this is the Globex
+  `security_desc`, populated from the definition's symbol/tag 55.
+- `IDGenerator` (newer): derives the namespace from new domain enums and uses
+  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
+  identifier; Databento deliberately passes the decimal instrument ID, and the
+  Python bridge uses the exchange security ID.
+
+### Logical-row examples
+
+The following names are illustrative. Production uses the exact string emitted
+by the selected feed/adapter, including its case, spacing, and weekly-series
+notation.
+
+#### Legacy `SecurityIDRecorder`
+
+For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
+Globex `security_desc`. After converting the fixed security to
+`api::Security`, `securities::descriptor()` returns that same
+`glbx.security_desc()` string.
+
+| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
+|---|---|---|---|
+| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
+| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
+| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
+| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
+
+A weekly option does not get a special namespace. Standard and weekly options
+are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
+`security_desc` distinguishes the weekly series. For example, if the database
+assigned rowids `42001` through `42004` to the rows above, those integers—not
+the descriptor strings—would be the Spark security keys.
+
+Conceptually, legacy allocation is:
+
+```text
+future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
+option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
+weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
+spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
+```
+
+#### Newer `IDGenerator`
+
+The namespace result is similar, but the third argument is chosen by the
+adapter. Examples using a description-oriented adapter are:
+
+```cpp
+get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
+// store: ns="GLBX.FUTURE", key="ESZ6"
+
+get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
+// store: ns="GLBX.OPTION", key="ESZ6 C6500"
+
+get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
+// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
+
+get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
+// store: ns="COINBASE.SPOT", key="BTC-USD"
+```
+
+Databento intentionally produces a different logical row even when the human
+description is `ESZ6`. If its instrument ID were `12345678`, it would call the
+equivalent of:
+
+```cpp
+get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
+// store: ns="GLBX.FUTURE", key="12345678"
+```
+
+Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
+`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
+while a provider that sends `exchange_security_id="BTCUSD"` produces a
+different row and therefore a different Spark key.
+
+These examples show why the tuple is only logical when the adapter's naming
+contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
+describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
+
+Both enum-prefix removals currently produce the same namespace spelling (for
+example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
+rowid space. Live rows cannot share a rowid. The table does not declare
+`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
+SQLite to reuse a number later. Security-key rows should therefore be treated
+as append-only operational data.
+
+`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
+and feed latency over maximum durability against a host/power failure. The
+unique `(ns, key)` index prevents two different rows for the same logical
+security.
+
+### Other places the key is stored
+
+| Location | Purpose | Authoritative for allocation? |
+|---|---|---|
+| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
+| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
+| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
+| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
+| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
+
+The `SecurityIDRecorder::work_db()` comment says a published security is also
+recorded in the security database, but that function does **not** write
+`securities.sqlite` in the current code. Full security definitions are persisted
+separately by `SecurityCache::flush_to_disk()` through
+`db::insert_or_replace()`.
+
+## CME MDP3 / OnixS pipeline in detail
+
+### 1. Construction and thread setup
+
+`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
+`MDProducer`. It constructs the recorder with:
+
+- a lookup callback that maps `ExchangeKey` through the CME index to the
+  handle's current Spark key; and
+- `Dispatcher::current()`, which drives a 200 ms recorder timer.
+
+In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
+dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
+then calls that thread's dispatcher `run()`. The older `md run onixs` command
+uses its command/dispatcher thread similarly.
+
+The OnixS feed engine has its own configured thread pool. Instrument and market
+data callbacks therefore originate on OnixS feed-engine threads, while recorder
+database work and final key assignment execute on the MDP3 dispatcher thread.
+
+| MDP3 stage | Executing thread | DB call? |
+|---|---|---|
+| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
+| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
+| Add CME-ID index entry | Same OnixS callback thread | No |
+| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
+| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
+| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
+
+### 2. Definition callback: exchange ID is set, Spark key is not
+
+For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
+
+```cpp
+fsec.numeric_id = def.securityId();
+```
+
+It also copies symbol/descriptor/venue and pricing fields. A newly constructed
+`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
+does not change it.
+
+For a spread, every leg starts with:
+
+```cpp
+leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
+leg.key = invalid_security_key;
+```
+
+Thus both the parent and legs initially carry exchange-routing identity, but no
+Spark identity.
+
+### 3. Shared-memory allocation and exchange indexing
+
+`MDP3Listener::add_security()` performs these operations in order:
+
+1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
+   cold entry and copies the security into the cold segment.
+2. Because the key is `0`, `MDProducer` queues the handle through
+   `SecurityIDRecorder::ensure_key()`.
+3. `_index.add_security(security.numeric_id, handle)` immediately creates the
+   CME-ID-to-handle mapping.
+4. The initial exchange status is written through the handle.
+
+This ordering means incoming books can already be routed by CME ID while the
+Spark key is pending. Shared-memory consumers cannot discover the security yet.
+Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
+when `key() != 0`, so a trade received in this short window updates the cold
+last-trade state but is not added to the keyed trade log.
+
+### 4. Recorder tick: spreads, handles, then database
+
+Every 200 ms the recorder calls, in this exact order:
+
+1. `work_spreads()` tries to fill previously unresolved spread leg keys by
+   asking `CMESecurityIndex` for each leg exchange ID.
+2. `work_handles()` drains new handles. Outrights are converted to an API
+   security and queued for DB lookup immediately. A spread without all leg keys
+   is retained in `pending_handle` and its leg pointers are registered in
+   `unresolved_legs`.
+3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
+
+Because spread work precedes new-handle work, a newly seen unresolved spread
+normally waits until at least the next timer tick to resolve legs. It waits
+longer if any outright leg has not yet received its Spark key. A spread's own
+key is not requested until every leg key is non-zero.
+
+### 5. Lookup or generation
+
+`work_db()` first queries `(ns, key)`:
+
+- Existing row: use its `rowid` as the key and invoke the callback.
+- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
+  rowid inside a transaction, the row object receives that ID, and the callback
+  runs after the transaction completes.
+
+New inserts are throttled to roughly 200 per timer pass (the current `> 200`
+check can admit 201) to avoid delaying heartbeats during startup. The queues are
+TBB concurrent queues, but all SQLite access is serialized onto the recorder's
+dispatcher thread. `check_access()` asserts this single-thread rule after the
+first work pass records the dispatcher thread ID.
+
+### 6. Final assignment and visibility
+
+The queued callback captures a copy of `MDProducerHandle` and calls
+`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
+then increments `security_count`. From this point:
+
+- shared-memory scans report the security;
+- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
+- trades and RFQs can be emitted with a non-zero key;
+- resolved spreads can copy this key into their leg structures.
+
+### 7. Sunday preload is a fast path
+
+For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
+definitions from `securities.sqlite`. It converts them with
+`set_security_key=true`, so both the security key and spread leg keys come from
+the serialized API security. `MDProducer::add_security()` calls `set_key()`
+immediately, and no recorder lookup is required for that preloaded handle. The
+CME exchange ID is still restored into `numeric_id`, allowing the normal CME
+index to be built.
+
+## Generalized/newer pipeline in detail
+
+### 1. Topology and ownership
+
+`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
+`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
+`MarketDataGateway`, and starts the configured providers. For each `(venue,
+source)` returned by a provider, the gateway creates an `MdSink` containing:
+
+- a venue-local `GeneralSecurityIndex`;
+- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
+- a `GeneralMarketDataListener` connecting the source to that producer/index.
+
+The source decides its callback thread. Some sources call the listener while
+starting; others own network or replay threads. The generalized listener calls
+the producer inline on that source thread. The process-wide `IDGenerator`
+serializes its own calls with a mutex, but the listener's transient
+`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
+expected to serialize security/event callbacks.
+
+| Generalized stage | Executing thread | DB call? |
+|---|---|---|
+| Build/parse domain definition | Source/adaptor thread | No |
+| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
+| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
+| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
+| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
+| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
+
+There are two important compatibility exceptions inside the provider framework:
+
+- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
+  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
+  legacy flow documented above.
+- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
+  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
+  producer/index. Its assignment is still synchronous and pre-populated, so it
+  resembles the newer ID flow even though it does not pass through
+  `GeneralMarketDataListener`.
+
+### 2. How a new ID is constructed
+
+For a normal live generalized adapter, ID creation happens **in the adapter,
+before** `GeneralMarketDataListener::on_securities()`:
+
+```text
+logical identity = (domain venue, domain security type, adapter-chosen name)
+Spark ID         = SQLite rowid for that identity in key_ids.sqlite
+```
+
+`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
+the enum prefixes:
+
+```text
+VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
+```
+
+It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
+row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
+next rowid. That integer is wrapped in `domain::SecurityID` and placed in
+`SecurityCommon::id`.
+
+The third identity component is adapter-specific, and this matters when trying
+to reproduce a mapping:
+
+| Source/adaptor family | Typical `name` sent to `IDGenerator` |
+|---|---|
+| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
+| Taifex | product/instrument ID string |
+| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
+| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
+| Emumba Deribit | extracted Emumba security descriptor |
+| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
+
+So “security name” alone is not a universal logical key. Reproduction requires
+the exact venue enum, exact security-type enum, and the exact adapter-specific
+name used at insertion time.
+
+### 3. What `GeneralMarketDataListener` does with that ID
+
+`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
+behavior is controlled by:
+
+```cpp
+const bool use_given_sec_key = true;
+```
+
+Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
+call the DB. It copies `security_common.id.value()` into its transient
+`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
+`_add_spot()` then calls `_copy_security_common()`, which sets:
+
+```cpp
+to.key = incoming_id;
+to.numeric_id = incoming_id;
+```
+
+This collapses the generalized source/correlation ID and Spark shared-memory
+key into the same number. The source's text `exchange_sec_id` remains on the
+domain security but is not copied into `FixedSecurity::numeric_id` by this
+path.
+
+`_commit_security()` then:
+
+1. calls `MDProducer::add_security()` with an already non-zero key;
+2. causes `MDProducerHandle::set_key()` to run immediately during that call;
+3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
+   number); and
+4. returns with the security already visible in shared memory.
+
+Because the sink's producer has no recorder, supplying `id == 0` would allocate
+the shared-memory entry but never assign or publish its key. Generalized live
+sources must therefore provide a valid non-zero ID before `on_securities()`.
+
+The listener contains an alternative branch for `use_given_sec_key == false`.
+That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
+sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
+also effectively unused. Turning the flag off is not a mechanical change: an
+adapter such as Databento allocates using numeric instrument ID while the
+listener branch would use `sec.desc()`, potentially creating a second key for
+the same instrument.
+
+There is also a spread failure edge case. `_find_sec(sec, true)` records the
+parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
+ID is absent, `_add_spread()` logs and returns without committing the spread. A
+later repeat of the parent definition is then classified as a duplicate and
+ignored, so definition ordering is a hard requirement in the current code.
+
+### 4. How later books, trades, and statistics find the security
+
+Every later domain event carries a `domain::SecurityID`:
+
+- `Book::security_id()`;
+- `TradeRecord::sec_id`; or
+- `SecurityMarketDataStatistics::security_id`.
+
+The listener first uses `cc_sec_id_map` to translate that event ID to the value
+stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
+`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
+but it still serves as the admission check that the definition was seen.
+
+Most high-throughput adapters avoid SQLite on market-data events:
+
+- Databento allocates on the definition and caches
+  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
+- Taifex keeps the resolved ID in its security-details mapper.
+- Other native subscriptions maintain comparable descriptor/ID maps.
+
+The Python bridge is a notable exception: `to_bts_security_()`,
+`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
+`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
+one SQLite `SELECT` for each converted Python definition, trade, or book. It
+returns the existing ID after the first insert, but it is still a DB call and
+takes the global `IDGenerator` mutex on every event.
+
+### 5. Persistence timing and latency
+
+The newer allocator is synchronous:
+
+- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
+  thread.
+- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
+  (SQLite autocommit), also on the caller's thread.
+- Listener publication: no further DB call under the current
+  `use_given_sec_key` setting.
+
+The ID therefore exists in `key_ids.sqlite` before the adapter sends the
+definition to the listener. There is no intentional 200 ms publication delay,
+but the source callback/startup path directly pays DB and mutex latency. Bulk
+definition discovery performs one lookup and possibly one insert per security;
+`IDGenerator` does not batch those operations.
+
+WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
+not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
+occasional file-system work. The impact is usually a startup/definition cost
+when adapters cache the result, but it becomes hot-path latency for code such as
+the current Python bridge that resolves every event through the DB.
+
+`GeneralMarketDataListener` itself does not persist the full contract definition
+to `securities.sqlite`; that remains the separate security-cache/metadata
+persistence path described earlier.
+
+## Comparison: MDP3 recorder vs generalized allocator
+
+| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
+|---|---|---|
+| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
+| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
+| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
+| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
+| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
+| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
+| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
+| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
+| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
+| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
+| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
+| Batching | Yes for new rows | No |
+| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
+
+Neither algorithm generates a UUID or a content hash. Both ultimately use an
+opaque, database-assigned SQLite rowid. It is normally increasing and looks like
+a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
+used to *look up* the row is logical; the resulting number is not derivable from
+that tuple.
+
+## Can another system reconstruct the same keys?
+
+Not from `(venue, type, name)` alone. Another system can reconstruct the
+**logical lookup tuple**, but it cannot calculate the numeric Spark key because
+the key depends on the historical insertion order of every row in
+`key_ids.sqlite`.
+
+For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
+row and receive different IDs if other securities were inserted before it on
+one system. They are sequential opaque IDs, not deterministic hashes and not
+UUIDs.
+
+Ways to preserve cross-system identity are:
+
+1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
+   identical. This preserves existing numeric keys but requires coordinated
+   allocation for new ones.
+2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
+   import rows with their original rowids before either system allocates new
+   securities.
+3. Treat a stable external identity (venue + type + exchange instrument ID, or
+   another canonical instrument identifier) as the cross-system key and carry a
+   translation table to each system's local Spark key.
+4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
+   be computed independently, but collision handling, descriptor
+   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
+   with orders/recordings/databases all need explicit design.
+
+Copying only `securities.sqlite` is insufficient as an allocator strategy. It
+contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
+the mapping used to find or allocate those IDs. Conversely, copying only
+`key_ids.sqlite` preserves the numbers but not all contract metadata.
+
+## Other assignment paths in `sparkd`
+
+The two main pipelines reduce to four ways a producer handle can acquire a key:
+
+1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
+   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
+   Sunday preload, relay playback, and other replay paths use this mechanism.
+2. **Recorder-backed live input.** The input key is zero and the producer owns a
+   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
+   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
+   `numeric_id` and otherwise follows the same recorder/index design.
+3. **Service assignment.** `SyntheticSecurityService` calls
+   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
+   callback receives a copied `api::Security` with its top-level `id` set, then
+   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
+   rowid allocator.
+4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
+   never becomes public. This is not a valid production assignment path, but it
+   is the failure mode if a generalized source omits its ID.
+
+Relay and replay deserve care when reasoning about persistence:
+
+- The generalized `MDReplayer` forwards the recorded domain security ID and
+  emits later books/statistics with the recorded `sec_key`, so the listener's
+  current use-given mode preserves the recording's numeric identity.
+- The legacy relay producer converts a recorded `api::Security` with
+  `set_security_key=true`, so its shared-memory publication also preserves the
+  embedded key.
+- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
+  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
+  separate mapping population step, not a deterministic derivation of the key.
+
+## Operational implications
+
+- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
+  definition continuity matter.
+- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
+  another host, even when all instruments are identical.
+- Before moving a feed between adapters, compare the exact `name` each adapter
+  supplies to `IDGenerator`. Matching venue/type is insufficient.
+- Cache generated IDs in the source after definition processing. Avoid calling
+  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
+  this guideline.
+- Keep generated rowids below the 32-bit shared-memory ceiling until the
+  `security_key` ABI is widened or explicit bounds checking is added to the
+  generalized path.
+- Ensure spread legs are announced before generalized spread parents.
+- Treat `synchronous=0` as a durability decision. A copied DB should be taken
+  with SQLite-aware backup/checkpoint handling rather than by casually copying
+  active WAL files.
+
+Useful inspection queries (run against the configured DB path) are:
+
+```sql
+SELECT rowid AS spark_key, ns, key
+FROM store
+ORDER BY rowid;
+
+SELECT rowid AS spark_key, ns, key
+FROM store
+WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
+```
+
+## Source map
+
+- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
+  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
+  and spread-leg resolution.
+- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
+  synchronous generalized lookup/allocation and process-wide mutex.
+- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
+  location, schema, WAL/synchronous pragmas, individual inserts, and batch
+  inserts.
+- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
+  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
+  shared-memory allocation, recorder handoff, and ordered publication.
+- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
+  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
+  widths and hot/cold storage fields.
+- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
+  and
+  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
+  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
+- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
+  producer/index insertion order.
+- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
+  current dedicated MDP3 dispatcher-thread ownership.
+- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
+  and
+  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
+  generalized source/sink topology, incoming-ID mapping, fixed-security
+  conversion, and publication.
+- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
+  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
+  and
+  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
+  representative adapter-side allocation and caching behavior.
+- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
+  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
+  persistence of full security definitions.
+- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
+  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
+  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
+  translation followed by Spark-keyed market-data consumption and MBO access.
+- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
+  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
+  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
+  representative downstream use of Spark security keys.
+- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
+  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
+  ID reuse.
@@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y ca-certificates
 RUN apt-get update && apt-get install -y libssh2-1-dev
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-get update && apt-get install -y sudo
+RUN apt-get update && apt-get install -y tzdata

 # GCC
 ARG gcc_major=13
@@ -18,6 +19,11 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
  && apt-get install -y dotnet8

+RUN apt-get update && apt-get install -y gdb
+RUN apt-get update && apt-get install -y fish
+RUN apt-get update && apt-get install -y vim
+RUN apt-get update && apt-get install -y less
+
 RUN locale-gen en_US.UTF-8

 ARG docker_user="bts"
@@ -0,0 +1,11 @@
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+  fish \
+  less \
+  vim
+
+USER bts
+WORKDIR /home/bts
@@ -49,14 +49,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{00D21F82-61F4-467F-B125-268018A185E5}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{513A3F0D-46B5-4AC2-B700-8671933F6347}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{D419D996-9870-4FF5-82AD-08370D1C945F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -283,102 +275,6 @@ Global
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.ActiveCfg = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.Build.0 = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.ActiveCfg = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.Build.0 = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.ActiveCfg = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.Build.0 = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.Build.0 = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.ActiveCfg = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.Build.0 = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.ActiveCfg = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.Build.0 = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.Build.0 = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.ActiveCfg = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.Build.0 = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.ActiveCfg = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.Build.0 = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.Build.0 = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.ActiveCfg = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.Build.0 = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -406,9 +302,5 @@ Global
 		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
 		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{00D21F82-61F4-467F-B125-268018A185E5} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{513A3F0D-46B5-4AC2-B700-8671933F6347} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{D419D996-9870-4FF5-82AD-08370D1C945F} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 	EndGlobalSection
 EndGlobal
@@ -102,10 +102,10 @@ namespace BCG
         List<ColumnInfo> col_info = new List<ColumnInfo>();
         col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
         foreach (Pair<string, Type> p in group_by_col_info)
-            col_info.Add(new ColumnInfo(p.first ?? string.Empty, col_index++, ExceptionAssert.NotNull(p.second)));
+            col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
         foreach (string tc_prefix in tc_prefixes)
            foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, ExceptionAssert.NotNull(p.second)));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
         IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
         if (_schema == null)
            _schema = schema;
@@ -182,7 +182,7 @@ namespace BCG
         if (_transpsed_col_value_comparer_wrapper != null)
            tc_values.Sort(_transpsed_col_value_comparer_wrapper);
         List<string> prefixes = new List<string>();
-         foreach (Pair<int, object?[]> p in tc_values)
+         foreach (Pair<int, object[]> p in tc_values)
            prefixes.Add(tc_prefixes[p.first]);
         _set_schema(prefixes, group_by_col_info, table_name);
         BlueTable result = new BlueTable(_schema);
@@ -1,17 +1,17 @@
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
-using System.Diagnostics.CodeAnalysis;
 using System.Text;

+using BCG;
+using BCG.Utilities;
+
 namespace BCG
 {
   public class ConcatenateColumnsMapper : IStreamMapper
   {
      protected IDictionary myTempColMap = new SortedList ();
-      protected ArrayList? myColMap;
-      protected BlueTableSchema? mySchema;
+      protected ArrayList myColMap;
+      protected BlueTableSchema mySchema;

      public ConcatenateColumnsMapper(params string [] columnDefs)
      {
@@ -55,7 +55,11 @@ namespace BCG
            myColMap = new ArrayList (myTempColMap.Count);
            foreach (DictionaryEntry de in myTempColMap)
            {
-               myColMap.Add(Array.ConvertAll((string[])de.Value!, c => inputSchema[c].colNo));
+               string[] cols = de.Value as string[];
+               ArrayList temp = new ArrayList ();
+               foreach (string col in cols)
+                  temp.Add (inputSchema[col].colNo);
+               myColMap.Add (temp.ToArray(typeof(int)) as int[]);
            }
            myTempColMap.Clear ();
         }
@@ -68,8 +72,6 @@ namespace BCG

      public virtual IBlueRow map (IBlueRow row)
      {
-         requireSchema();
-         requireColMap();
         object[] retVal = new object[mySchema.columnInfo.Length];
         row.CopyTo (retVal, myColMap.Count);
         for (int colNo = 0; colNo < myColMap.Count; colNo ++)
@@ -88,21 +90,11 @@ namespace BCG
      /// <returns></returns>
      protected virtual object myGetConcatValue (IBlueRow row, int colNo)
      {
-         requireColMap();
         StringBuilder b = new StringBuilder();
-         foreach (int cn in (int[])myColMap[colNo]!) {
-            b.Append(row[cn]?.ToString());
-         }
+         foreach (int cn in myColMap[colNo] as int[])
+            b.Append(row[cn].ToString());
         return b.ToString ();
      }
-
-      [MemberNotNull(nameof(myColMap))]
-      protected void requireColMap() =>
-         ExceptionAssert.NotNull(myColMap);
-
-      [MemberNotNull(nameof(mySchema))]
-      void requireSchema() =>
-         ExceptionAssert.NotNull(mySchema);
   }

   public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper 
@@ -115,21 +107,23 @@ namespace BCG
         myDicts = new HashTableND [columnDefs.Length];
         int i = 0;
         foreach (DictionaryEntry de in myTempColMap)
-            myDicts[i++] = new HashTableND(((string[])de.Value!).Length);
+            myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
      }

      protected override object myGetConcatValue (IBlueRow row, int colNo)
      {
-         requireColMap();
+         int[] cols = myColMap[colNo] as int[];
+         object[] vals = new object[cols.Length];
+         for (int i = 0; i < vals.Length; i++)
+            vals[i] = row[cols[i]];
         HashTableND ht = myDicts[colNo];
-         var vals = Array.ConvertAll((int[])myColMap[colNo]!, c => row[c]);
-         var rv = ht.getValue(vals);
+         object rv = ht.getValue (vals);
         if (rv != null)
            return rv;
         else
         {
            rv = base.myGetConcatValue (row, colNo);
-            var args = new object?[vals.Length + 1];
+            object[] args = new object[vals.Length + 1];
            vals.CopyTo (args, 0);
            args[args.Length-1] = rv;
            ht.add (args);
@@ -14,10 +14,8 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
         : base (colMapExpression, null, columnNames) {}

-
-      public override IBlueRow evaluate (DataPacket packet)
-      {
-         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
+      public override IBlueRow? evaluate(IDataPacket packet) {
+         myOutputTable ??= new GroupByOutputTable(packet.Schema);
         string hashString = GroupByStream.getRepString (packet, myGroupByCols);
         if (packet.newRows.Count == 0)
            //if there are no rows for this group we remove the group from the output table
@@ -67,7 +67,7 @@ namespace BCG {
      }
      #endregion

-      public event FilterChangeDelegate? onFilterChange;
+      public event FilterChangeDelegate onFilterChange;

      #region IStreamFilter
      public bool filter(IBlueRow row) {
@@ -1,4 +1,4 @@
-using Bts.DataStreamCore;
+using System;

 namespace BCG
 {
@@ -10,8 +10,8 @@ namespace BCG
 	{
      protected string[] myGroupByCols;
      protected string myOutputColumn;
-      protected GroupByOutputTable? myOutputTable;
-      protected string? myColMapExpression;
+      protected GroupByOutputTable myOutputTable;
+      protected string myColMapExpression;

      #region constructors
 		public GroupByEvaluator (string outputCol, string[] groupByCols)
@@ -42,7 +42,7 @@ namespace BCG
         return retVal;
      }

-      public abstract IBlueRow evaluate (DataPacket dp) ;
+      public abstract IBlueRow? evaluate(IDataPacket dp);
      #endregion

      #region Helper methods
@@ -52,20 +52,21 @@ namespace BCG
      /// <param name="r"></param>
      /// <param name="id"></param>
      /// <returns></returns>
-      protected IBlueRow? myProject(IBlueRow r, int id) =>
-         r is null
-            ? null
-            : myColMapExpression is null || myColMapExpression.Length < 1
-               ? BlueDataUtility.getNewReplacedRow(r, id, "id")
-               : BlueDataUtility.getNewReplacedRow(myProjectRow(r, myColMapExpression), id, "id");
-
+      protected IBlueRow myProject (IBlueRow r, int id)
+      {
+         return r == null ? null :
+            myColMapExpression == null || myColMapExpression.Length < 1 ?
+               BlueDataUtility.getNewReplacedRow (r, id, "id") :
+               BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression),
+            id, "id");
+      }
+      
      /// <summary>
      /// Projects the id, conserving the id
      /// </summary>
      /// <param name="r"></param>
      /// <returns></returns>
-      protected IBlueRow? myProject(IBlueRow r) =>
-         myProject(r, r is null ? int.MinValue : r.Id());
+      protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
      #endregion

      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
@@ -1,6 +1,5 @@
-using Bts.DataStreamCore;
-
 using System;
+using BCG.Utilities;
 using System.Collections;

 namespace BCG
@@ -23,12 +22,10 @@ namespace BCG
      /// <param name="groupRepString"></param>
      public void addRow (IBlueRow row, string groupRepString)
      {
-         var grp = myDict[groupRepString];
-         if (grp is not null) {
-            myTable.deleteRow((IBlueRow)grp);
-         }
+         if (myDict[groupRepString] != null)
+            myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
         myTable.addRow (row);
-         myDict[groupRepString] = myTable[row.Id()];
+         myDict[groupRepString] = myTable[(int)row["id"]];
      }

      /// <summary>
@@ -38,7 +35,7 @@ namespace BCG
      public void removeRow (string groupRepString)
      {
         // BlueEventLog.logAssert (myDict[groupRepString] != null);
-         myTable.deleteRow ((IBlueRow)myDict[groupRepString]!);
+         myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
         myDict.Remove (groupRepString);
      }

@@ -51,8 +48,11 @@ namespace BCG
         myDict.Clear ();
      }

-      public IBlueRow? getRow(string groupRepString) =>
-         myDict[groupRepString] as IBlueRow;
+      public IBlueRow getRow (string groupRepString)  
+      {
+         return myDict[groupRepString] == null ? null : 
+            myDict[groupRepString] as IBlueRow;
+      }

      #region IBlueRowCollection members
      public IBlueRow this[int id]  {get{return myTable[id];}}
@@ -1,21 +1,20 @@
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
+using BCG.Utilities;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;

 namespace BCG {
   internal class GroupByStream : MethodDrivenStream {
      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
-         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         readonly Func<IDataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
            _groupby_cols = groupby_cols.ToArray();
            _groupby_func = groupby_func;
         }
-         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
+         public IBlueRow evaluate(IDataPacket packet) =>
+            _groupby_func(packet);

         public string[] groupByColumns { get { return _groupby_cols; } }

@@ -27,15 +26,14 @@ namespace BCG {
      /// Serves as  a quick way to access all rows belonging to the same group
      /// </summary>
      private bool myIdGuard = true;
-      RowDictionaryTable? myBookKeepingTable;
+      private RowDictionaryTable myBookKeepingTable;
      private IGroupByEvaluator myGroupByEvaluator;
-      // input schema, as output schema can differ (IGroupByEvaluator's
-      IBlueTableSchema? mySchema;
+      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
      //can map a schema to another
-      IBlueTableSchema? myOutputSchema;
+      private IBlueTableSchema myOutputSchema;
      private int myIdGenerator = 0;

-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }

      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -48,14 +46,14 @@ namespace BCG {
      }

      protected override void myHandleFirstCall() {
-         requireLastArrivedPacket();
         mySchema = myLastArrivedPacket.schema;
         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
            myGroupByEvaluator.groupByColumns);
         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            var newRow = myEvaluate(new DataPacket(rowGroup, rowGroup, new BlueTable(mySchema)));
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
            if (newRow != null)
               temp.addRow(newRow, myGetGroupString(rowGroup));
         }
@@ -81,14 +79,15 @@ namespace BCG {
         ArrayList handledAddGroups = new ArrayList();
         string groupRepString;

-         requireBookKeeping();
         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
            // BlueEventLog.logAssert (delGroup.Count > 0);
            groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) ?? new BlueTable(mySchema);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
            handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) ?? new BlueTable(mySchema);
-            var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
            myFireSwitch |= myMergeRow(newRow, groupRepString);
         }

@@ -100,16 +99,16 @@ namespace BCG {
               //the loop above

               groupRepString = myGetGroupString(addGroup);
-               var aggGroup = ExceptionAssert.NotNull(myBookKeepingTable.getGroup(groupRepString));
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
               // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, new BlueTable(aggGroup.schema)));
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
               myFireSwitch |= myMergeRow(newRow, groupRepString);
            }
         }
      }

-      private bool myMergeRow(IBlueRow? newRow, string groupRepString) {
-         requireLastFiredPacket();
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
         //BlueEventLog.logAssert (newRow != null || existingRow != null);
         if (existingRow == null && newRow == null)
@@ -117,7 +116,6 @@ namespace BCG {
         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
            return false;
         else if (existingRow == null) {
-            ArgumentNullException.ThrowIfNull(newRow);
            //row added for the first time
            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
@@ -140,9 +138,6 @@ namespace BCG {
      /// <param name="delGroups"></param>
      /// <param name="addGroups"></param>
      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
-         ExceptionAssert.NotNull(myBookKeepingTable);
-         requireLastFiredPacket();
-         requireLastArrivedPacket();
         delGroups =
            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
         addGroups =
@@ -156,19 +151,16 @@ namespace BCG {
      }

      protected override void myOnTableClear() {
-         if (ExceptionAssert.NotNull(myBookKeepingTable).Count == 0) {
-            // ie there arent any rows, so no need to optimize the 'cleansing' process
-            return;  
-         }
-         requireLastFiredPacket();
+         if (myBookKeepingTable.Count == 0)
+            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         ((GroupByOutputTable)myLastFiredPacket.newRows).clear();
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
         myBookKeepingTable.clear();
         myGroupByEvaluator.clear();
      }

-      private IBlueRow? myEvaluate(DataPacket dp) {
+      private IBlueRow myEvaluate(DataPacket dp) {
         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
         return evalRow == null ? null :
            myIdGuard ?
@@ -196,11 +188,11 @@ namespace BCG {
         }
      }

-      public static string getRepString(DataPacket dp, string[] colNames) {
+      public static string getRepString(IDataPacket dp, string[] colNames) {
         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
      }

-      public static IBlueRow? getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow(IDataPacket dp) {
         if (dp.newRows.Count > 0)
            return BlueDataUtility.extractRow(dp.newRows, 0);
         else if (dp.addedRows.Count > 0)
@@ -219,13 +211,13 @@ namespace BCG {
      /// <param name="newValue"></param>
      /// <param name="isAdd"></param>
      /// <returns></returns>
-      public static object? incrementalAdd(object lastValue, object newValue, bool isAdd) {
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
         if (lastValue == null || lastValue.Equals(DBNull.Value))
            return newValue;
         // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
-         object? retVal = null;
+         Object retVal = null;
         if (newValue.GetType().Equals(typeof(int)))
            retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
         else if (newValue.GetType().Equals(typeof(Double)))
@@ -237,9 +229,6 @@ namespace BCG {
         return retVal;
      }

-      [MemberNotNull(nameof(myBookKeepingTable))]
-      void requireBookKeeping() =>
-         ExceptionAssert.NotNull(myBookKeepingTable);
   }

 }
@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
   /// <summary>
@@ -14,7 +12,7 @@ namespace BCG
      /// deletetion) and the added and deletedRows being the increments. If a group has
      /// to be totally removed then a null should be returned
      /// </summary>
-      IBlueRow evaluate (DataPacket packet);
+      IBlueRow? evaluate(IDataPacket packet);

      /// <summary>
      /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator
@@ -14,9 +14,8 @@ namespace BCG {
        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
           : base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
@@ -1,9 +1,10 @@
+using System;
+using System.Collections;
+using BCG.Utilities;
+using System.IO;
+
 namespace BCG
 {
-   using Spark.Core.Diagnostics;
-
-   using System.Diagnostics.CodeAnalysis;
-
   /// <summary>
   /// Some basic event handling functionality factorred out of any Stream whose behaviour
   /// is driven by some interface passed on by the user
@@ -11,8 +12,8 @@ namespace BCG
   public abstract class MethodDrivenStream : ViewStream
   {
      protected ViewStream myStream;
-      protected DataPacket? myLastArrivedPacket;
-      protected DataPacket? myLastFiredPacket;
+      protected DataPacket myLastArrivedPacket;
+      protected DataPacket myLastFiredPacket;
      protected bool myFireSwitch = true;

      public MethodDrivenStream(ViewStream vs)  {myStream = vs;}
@@ -58,7 +59,6 @@ namespace BCG
      /// To handle the first call, the LastFiredPacket is supposed to be initialized and filled
      /// with data.
      /// </summary>
-      [MemberNotNull(nameof(myLastFiredPacket))]
      protected abstract void myHandleFirstCall ();

      /// <summary>
@@ -89,13 +89,5 @@ namespace BCG
      {
         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
      }
-
-      [MemberNotNull(nameof(myLastArrivedPacket))]
-      protected void requireLastArrivedPacket() =>
-         ExceptionAssert.NotNull(myLastArrivedPacket);
-
-      [MemberNotNull(nameof(myLastFiredPacket))]
-      protected void requireLastFiredPacket() =>
-         ExceptionAssert.NotNull(myLastFiredPacket);
   }
 }
@@ -1,7 +1,3 @@
-using System;
-
-using BCG.Utilities;
-
 namespace BCG {
    public class MinGroupByEvaluator : MaxGroupByEvaluator {
        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -10,8 +6,8 @@ namespace BCG {
        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
           base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG {
    /// <summary>
    /// Computes the sum for a given column
@@ -10,8 +8,8 @@ namespace BCG {
        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
           : base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);

            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
            return sumRow;
@@ -33,7 +31,7 @@ namespace BCG {

        #region IGroupByEvaluator Members

-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);

        public string[] groupByColumns { get { return myGroupByCols; } }

@@ -51,9 +49,8 @@ namespace BCG {
        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
           : base(colMapEpression, outputCol, groupByCols) { }

-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
            return sumRow;
        }
@@ -68,7 +65,7 @@ namespace BCG {

        #region IGroupByEvaluator Members

-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);

        public string[] groupByColumns { get { return myGroupByCols; } }

@@ -1,7 +1,5 @@
 using mbts;

-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections.Generic;

@@ -37,7 +35,7 @@ namespace BCG {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
         foreach (Triplet<ViewStream, string, string> r in join_specs) {
            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = ExceptionAssert.NotNull(r.third).Split(' ');
+            string[] visible_cols = r.third.Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
@@ -49,7 +47,7 @@ namespace BCG {
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add(new(ExceptionAssert.NotNull(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
         return rv.ToArray();
      }
@@ -290,7 +290,6 @@ namespace BCG
               }
            }
         }
-         ExceptionAssert.NotNull(myState);
      }

      private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
@@ -1,5 +1,3 @@
-using Spark.Core.Diagnostics;
-
 using System;

 namespace BCG
@@ -20,21 +18,16 @@ namespace BCG

      public override void onTableClear (bool isLeft)
      {
-         requireSchema();
-         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
         myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
         myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft) {
-            ExceptionAssert.NotNull(myLeftDataPacket);
+         if (!isLeft)
            foreach (IBlueRow row in myLeftDataPacket.newRows)
               ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
-         }
      }

      protected override void myInitFiredPacket ()
      {
-         requireSchema();
         myLastFiredPacket = new DataPacket ();
         myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
         myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -15,7 +15,7 @@ namespace BCG
         : this (r, null, null)
      {}

-      public SimpleNode(IBlueRow? r, SimpleNode? nx, SimpleNode? pv) {
+      public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
         row = r;
         next = nx;
         prev = pv;
@@ -1,7 +1,5 @@
 using bts;

-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections.Generic;

@@ -62,7 +60,7 @@ namespace BCG {
            if (columnAndType.Length == 1)
               ci[n] = new ColumnInfo(c, n, typeof(double));
            else
-               ci[n] = new(columnAndType[0], n, ExceptionAssert.NotNull(Type.GetType("System." + columnAndType[1], true)));
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
            n++;
         }

@@ -2,6 +2,8 @@ using BCG;

 using MongoDB.Driver.Core.WireProtocol.Messages;

+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -37,7 +39,7 @@ namespace bts {
      }

      void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
         IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
         :
@@ -23,6 +23,6 @@ namespace BCG
      /// fires an event when the mapper 
      /// changes itself
      /// </summary>
-      event MapperChangeDelegate? onMapperChange;
+      event MapperChangeDelegate onMapperChange;
   }
 }
@@ -75,7 +75,6 @@ namespace BCG {

      protected override void myHandleFirstCall() {
         ArgumentNullException.ThrowIfNull(myMapper);
-         requireLastArrivedPacket();
         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
         myLastFiredPacket = new DataPacket();
         myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -86,12 +85,10 @@ namespace BCG {
      }

      protected override void myHandleIncrementalAdd() {
-         requireLastArrivedPacket();
         if (myLastArrivedPacket.addedRows.Count == 0) {
            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
            return;
         }
-         requireLastFiredPacket();
         ArgumentNullException.ThrowIfNull(myMapper);
         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
@@ -104,12 +101,10 @@ namespace BCG {
      }

      protected override void myHandleIncrementalDelete() {
-         requireLastArrivedPacket();
         if (myLastArrivedPacket.deletedRows.Count == 0) {
            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
            return;
         }
-         requireLastFiredPacket();
         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
@@ -132,7 +127,6 @@ namespace BCG {
      }

      protected override void myOnTableClear() {
-         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
@@ -47,14 +47,14 @@ namespace BCG {
            _map_rows = map_rows;
         }

-         event MapperChangeDelegate? ISuperStreamMapper.onMapperChange { add { } remove { } }
-
         public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }

         public IBlueRow[] map(IBlueRow r) {
            IEnumerable<IBlueRow> rows = _map_rows(r);
            return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
         }
+
+         public event MapperChangeDelegate? onMapperChange;
      }

      //maps the id of the underlying stream to the
@@ -608,8 +608,8 @@ namespace BCG {
            // 5. deserialize the state into a stream op
            str_io_msg_p = deser.deserialize(content_p);
            // 6. check to make sure the state's rows are good
-            passes &= str_io_msg_p.op == StreamIOMessage.StreamOp.CREATE
-               && BlueDataUtility.compute_equality(cr_rows, ExceptionAssert.NotNull(str_io_msg_p.create_message).rows);
+            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
+                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
            return new TestResult {
                test_result = passes,
                test_name = "run_pubsub_server_stream_tag_handler_test"
@@ -692,7 +692,7 @@ namespace BCG {
            var content = st_data.get_current_content();
            var msg_p = deser.deserialize(content);
            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
-            return ExceptionAssert.NotNull(msg_p.create_message).rows;
+            return msg_p.create_message.rows;

        }

@@ -1,7 +1,5 @@
 using Bts.ColumnSortExpressions;

-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -41,7 +39,7 @@ public class SortRow : IComparable, IBlueRow {
         var otherVal = otherRow[col.Name];
         int rv = myIsNull(this[col.Name]) ?
            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ExceptionAssert.Argument.Is<IComparable>(this[col.Name]).CompareTo(otherVal));
+            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
         rv *= col.IsAscending ? 1 : -1;
         return rv;
      }
@@ -1,6 +1,4 @@
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;

@@ -62,13 +60,13 @@ namespace BCG
            List<string> columnInfoStrParts = new List<string>(columnInfoStr.Split(new char[] { ':' }, StringSplitOptions.None));
            if (columnInfoStrParts.Count != 2)
            {
-               throw new Exception($"Unable to decode schema string '{schemaStr}'");
+               throw (new Exception(String.Format("Unable to decode schema string '{0}'", schemaStr)));
            }

-            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, ExceptionAssert.NotNull(Type.GetType(columnInfoStrParts[1]))));
+            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, Type.GetType(columnInfoStrParts[1])));
         }

-         return new BlueTableSchema(columnInfos.ToArray(), string.Empty);
+         return (new BlueTableSchema(columnInfos.ToArray(), ""));
      }

      private IBlueRow myGetRow(IBlueTableSchema schema, string rowStr)
@@ -77,7 +75,7 @@ namespace BCG

         if (schema.columnInfo.Length != valueStrs.Count)
         {
-            throw new Exception($"Unable to decode row string '{rowStr}'");
+            throw (new Exception(String.Format("Unable to decode row string '{0}'", rowStr)));
         }

         List<object> values = new List<object>();
@@ -86,7 +84,7 @@ namespace BCG
            values.Add(Convert.ChangeType(valueStrs[i], schema.columnInfo[i].colType));
         }

-         return new BlueRow(schema, values.ToArray());
+         return (new BlueRow(schema, values.ToArray()));
      }
      #endregion

@@ -1,7 +1,6 @@
 using BCG.Utilities.General;

 using System;
-using System.Collections.Generic;
 using System.Threading;

 namespace BCG
@@ -98,7 +97,7 @@ namespace BCG
         // BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread, 
         //    "onTargetDoWork should only be called from target thread");
         var updates = _queued_updates.dequeueAll();
-         foreach (var u in updates.ExceptNull()) {
+         foreach (var u in updates) {
            if (u.subscription == _current_target_subscription)
               myFireEvent(u.dp);
         }
@@ -1,9 +1,7 @@
-using BCG.Utilities.General;
-
-using Spark.Core.Diagnostics;
-
 using System;

+using BCG.Utilities.General;
+
 namespace BCG {
   public abstract class TimeUnitStream : ViewStream, IWorker {
      protected TimeSpan myOffset;
@@ -48,8 +46,8 @@ namespace BCG {
            myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));


-         var latestRow = myGetLatestRow();
-         if (latestRow is null || !ExceptionAssert.Argument.Is<DateTime>(latestRow["time"]).Equals(dt)) {
+         if (myGetLatestRow() == null ||
+            !myGetLatestRow()["time"].Equals(dt)) {
            IBlueRow ar = new BlueRow(myCollection.schema,
               myId++, dt);
            IBlueRow dr = myCollection.addRowWithReturn(ar);
@@ -73,10 +71,11 @@ namespace BCG {

      #region Private Helper methods

-      IBlueRow? myGetLatestRow() =>
-         myCollection.Count == 0
-            ? null
-            : BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
+      private IBlueRow myGetLatestRow() {
+         return myCollection.Count == 0 ?
+            null :
+            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
+      }

      private IBlueRowCollection myConvertToCollection(IBlueRow r) {
         BlueTable retVal = new BlueTable(myCollection.schema);
@@ -1,42 +1,43 @@
-namespace BCG;
+using System;

-public class TopStream : MethodDrivenStream {
-   int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;

-   public TopStream(ViewStream stream, int topCount)
-      : base(stream) =>
-      myTopCount = topCount;
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }

-   public TopStream(ViewStream stream, int topCount, string sortExpression)
-      : this(sortExpression is null ? stream : new SortStream(stream, sortExpression), topCount) { }
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }

-   protected override void myOnTableClear() {
-      requireLastFiredPacket();
-      myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-      myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-   }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }

-   protected override void myHandleFirstCall() {
-      requireLastArrivedPacket();
-      myLastFiredPacket = new DataPacket();
-      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-      myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
-   }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }

-   protected override void myHandleIncrementalAdd() { }
+        protected override void myHandleIncrementalAdd() { }

-   protected override void myHandleIncrementalDelete() {
-      requireLastArrivedPacket();
-      requireLastFiredPacket();
-      IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-      BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-         oldRows,
-         myLastFiredPacket.schema.columnNames,
-         out myLastFiredPacket.deletedRows,
-         out myLastFiredPacket.addedRows);
-      myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-   }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
 }
@@ -1,10 +1,12 @@
+
+using System;
 using System.Collections;

 namespace BCG {

   public class AlphabetizeColumnsStream : ViewStream {
      private ViewStream mySrc;
-      IBlueTableSchema? mySchema = null;
+      private IBlueTableSchema mySchema = null;

      public AlphabetizeColumnsStream(ViewStream src) {
         mySrc = src;
@@ -1,12 +1,12 @@
-using System.Linq;
+using System;

 namespace BCG
 {
 	public class AppendStreamModified : ViewStream
 	{
      private AppendCollection myColl = new AppendCollection ();
-      ViewStream? myLeftStream, myRightstream;
-
+      private ViewStream myLeftStream, myRightstream;
+	
      public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
 		{
         myLeftStream = leftStream;
@@ -15,24 +15,11 @@ namespace BCG

      public AppendStreamModified (ViewStream[] vss)
      {
-         switch (vss.Length) {
-            case 0:
-               break;
-            case 1:
-               mySetRealSource (vss[0]);
-               break;
-            default:
-               mySetRealSource(
-                  vss[1..].Aggregate(
-                     new AppendStreamModified(new[] { vss[0] }),
-                     (asm, vs) => new AppendStreamModified(asm, vs)));
-               break;
-         }
         if (vss.Length == 1)
            mySetRealSource (vss[0]);
         else
         {
-            AppendStreamModified? rv = null;
+            AppendStreamModified rv = null;
            foreach (ViewStream s in vss)
               rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s); 
            mySetRealSource (rv);
@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;

+using global::Spark.Core.Disposables;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;

-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -8,6 +8,7 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;

+using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;

@@ -43,14 +43,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "..\cs\Bts.Collections\Bts.Collections.csproj", "{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "..\cs\Bts.Core\Bts.Core.csproj", "{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "..\cs\Bts.Disposables\Bts.Disposables.csproj", "{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "..\cs\Bts.Numerics\Bts.Numerics.csproj", "{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -301,54 +293,6 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.Build.0 = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.Build.0 = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.ActiveCfg = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.Build.0 = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.ActiveCfg = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.Build.0 = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.Build.0 = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.Build.0 = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.Build.0 = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.Build.0 = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.ActiveCfg = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.Build.0 = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.ActiveCfg = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.Build.0 = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.Build.0 = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.Build.0 = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.ActiveCfg = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.Build.0 = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.ActiveCfg = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -5,7 +5,7 @@ using BCG.Utilities.General;

 using Bts.DataStreamCore;

-using Spark.Core.Collections;
+using global::Spark.Core.Collections;

 public class TestDriverStream : ViewStream, IWorker {
   readonly IBlueTableSchema schema;
@@ -22,6 +22,7 @@ namespace BCG {
   /// with PubSub, as some clients are still using ManagedSubcribers)
   /// </summary>
   public class ManagedSubscriber : IManagedSubscriber, IWorker, IDisposable {
+      private static ManagedSubscriber? _self;
      private ISubscriberCore _subcore;
      // private IDictionary myTypedDispatchTable = new Hashtable(),
      //    myRawDispatchTable = new Hashtable();
@@ -33,6 +34,7 @@ namespace BCG {
      ManagedSubscriber(ISubscriberCore subsCore) { _subcore = subsCore; }

      public void doWork() {
+         var msg_map = new Dictionary<string, List<bts.PubSubServerMessage>>();
         var updates = _subcore.get_updates(TimeSpan.Zero);
         if (updates.Count() == 0)
            return;
@@ -45,7 +45,11 @@ namespace BCG
         get { return myConnInfo; }
      }

-      public override string ToString() =>
-         $"type: {myMsgType} conn: {myConnInfo} msg: {myMsg?.Length ?? 0}";
+      override public string ToString()
+      {
+         Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
+         return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
+      }
+
   }
 }
@@ -46,7 +46,7 @@ namespace bts {
            System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
            _accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { });
            if (ex is OutOfMemoryException)
-               throw;
+               throw ex;
            return;
         }
         new_sock.LingerState = new LingerOption(false, 0);
@@ -1,8 +1,9 @@
-using bts;
-
 using System;
 using System.IO;
-using System.Text;
+using System.Collections;
+using bts;
+using System.Threading;
+using System.Reflection;

 namespace BCG {
   /// <summary>
@@ -68,8 +69,7 @@ namespace BCG {

      public static byte[] getBytesFromStream(params Stream[] streams) {
         long totalSize = 0;
-         using var ms = new MemoryStream();
-         using var bw = new BinaryWriter(ms, Encoding.UTF8, true);
+         BinaryWriter bw = new BinaryWriter(new MemoryStream());
         foreach (Stream s in streams)
            totalSize += s.Length - s.Position;
         bw.Write(Convert.ToInt32(totalSize));
@@ -78,8 +78,7 @@ namespace BCG {
            int lengthToReadForThisStream = Convert.ToInt32(s.Length - s.Position);
            bw.Write(br.ReadBytes(lengthToReadForThisStream));
         }
-         bw.Flush();
-         return ms.ToArray();
+         return (bw.BaseStream as MemoryStream).ToArray();
      }

      public static byte[] read_bytes(Stream str) {
@@ -92,7 +92,7 @@ namespace BCG {

      public void publish(PubSubClientMessage msg) {
         Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
-         Log.assert(!string.IsNullOrEmpty(msg.pub_msg?.tag_type));
+         Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
         bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
         _main_queue.enqueue(msg);
      }
@@ -143,7 +143,7 @@ namespace BCG {
      public void doWork() {
         if (_connection_cb != null) {
            var conns = _conn_callback_queue.dequeueAll();
-            foreach (var ci in conns.ExceptNull()) {
+            foreach (var ci in conns) {
               _connection_cb(ci.client_url, ci.status);
            }
         }
@@ -175,7 +175,7 @@ namespace BCG {
            case RemoteMessageType.Data:
               // if callback is registered fire it right away, else enqueue it
               if (_data_cb != null)
-                  _data_cb(sender_rci.ToString(), message.message ?? Array.Empty<byte>());
+                  _data_cb(sender_rci.ToString(), message.message);
               else
                  _received_data_queue.enqueue(new Pair<string, byte[]>(sender_rci.ToString(), message.message));
               break;
@@ -258,7 +258,7 @@ namespace BCG {
      // (which gets called in another thread)
      private void _handle_new_connections() {
         var new_conns = _new_connections.dequeueAll();
-         foreach (var conn in new_conns.ExceptNull()) {
+         foreach (ITCPConnection conn in new_conns) {
            var peer_rci = conn.get_peer_address();
            bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections conn: {conn}");
            ///to stop it from being garbabge collected immediatly
@@ -16,8 +16,8 @@ public interface IPubSubState {

   ICollection<string> all_tags { get; }

-   byte[]? get_content(string tag);
-   string? getTagType(string tag);
+   byte[] get_content(string tag);
+   string getTagType(string tag);
   void get_all_connections(
      string tag,
      out Dictionary<string, string> subscribers,
@@ -188,11 +188,11 @@ namespace BCG {
         _state.handle_sub(sub_msg);   // state keeps the list of all subscribers
         ///to all these new subscriber s, send the existing data
         foreach (string tag in sub_msg.tags.empty_if_null()) {
-            byte[]? data = _state.get_content(tag);
+            byte[] data = _state.get_content(tag);
            //ie data has been published once
            if (data != null) {
               bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
-               string? tag_type_ = _state.getTagType(tag);
+               string tag_type_ = _state.getTagType(tag);
               var msg_to_send = new PubSubServerMessage() {
                  tag = tag,
                  tag_type = tag_type_,
@@ -148,7 +148,7 @@ namespace BCG {
            return new string[] { };
      }

-      public byte[]? get_content(string tag) {
+      public byte[] get_content(string tag) {
         if (!_tag_to_data_map.ContainsKey(tag))
            return null;
         var rv = _tag_to_data_map[tag].get_current_content();
@@ -157,7 +157,7 @@ namespace BCG {
         return rv;
      }

-      public string? getTagType(string tag) {
+      public string getTagType(string tag) {
         if (!_tag_to_data_map.ContainsKey(tag))
            return null;
         else
@@ -20,7 +20,7 @@ namespace bts {
   public class SingleTagData {
      private readonly IPubSubServerTagHandler? _tag_handler;

-      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler? tag_handler) {
+      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
         bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
         this.tag = tag;
         this.tag_type = tag_type;
@@ -46,6 +46,7 @@ namespace BCG {
      }

      public void onConnection(ITCPConnection connection) {
+         string ip; int port;
         var peer_rci = connection.get_peer_address();
         bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop");
         if (_listener != null) {
@@ -5,10 +5,9 @@ using BCG;
 using bts.api;
 using bts.md;

-using Bts.DataStream;
 using Bts.DataStreamCore;

-using Spark.Api.MarketData;
+using global::Spark.Api.MarketData;

 public class BookSchemaBuilder {
   public int NumLevels { get; set; } = 5;
@@ -2,7 +2,7 @@ namespace Bts.SharedStreams;

 using bts.api;

-using Spark.Api.Orders;
+using global::Spark.Api.Orders;

 public static class WellKnownColumns {
   public static (string Name, Type Type) AskNotional => (WellKnownColumnNames.AskNotional, typeof(double));
@@ -10,7 +10,7 @@
   [DataStreamUserRespresentation("nl")]
   public class BlueOrbNLStream : ViewStream {
      private readonly string _nl_prompt;
-      private ViewStream? _nl_to_dsl_stream, _dsl_stream;
+      private ViewStream _nl_to_dsl_stream, _dsl_stream;
      private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
      public BlueOrbNLStream(string nl_prompt) {
         _nl_prompt = nl_prompt;
@@ -85,8 +85,7 @@ namespace bts.BlueOrb {
                .GetProperty("choices")[0]
                .GetProperty("message")
                .GetProperty("content")
-                .GetString()
-                ?? string.Empty;
+                .GetString();

            //return gpt_response;
            DateTime t1 = DateTime.Now;
@@ -4,8 +4,6 @@ using BCG.Utilities.General;
 using Bts.PubSub;
 using Bts.SharedStreams.Tables;

-using Spark.Core.Diagnostics;
-
 namespace bts {
   // TODO: Use dependency injection to instantiate this stream, and others like it that depend on
   // ISubscriberCore(Factory). Until then, hide it from DSQA.
@@ -42,7 +40,7 @@ namespace bts {
         foreach (var u in updates) {
            if (u.tag != "System_AllStatistics")
               continue;
-            var tag_infos = TagStatsInfo.deserialize(ExceptionAssert.NotNull(u.data));
+            var tag_infos = TagStatsInfo.deserialize(u.data);
            foreach (var ti in tag_infos) {
               _tag_map[ti.tag] = ti;
            }
@@ -187,4 +187,24 @@
      Name="Test"
      DependsOnTargets="Build"></Target>

+  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
+    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
+    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
+  </Target>
+
+  <ItemGroup>
+    <Content Include="@(NativeWindowsAsset)"
+             Condition="Exists('@(NativeWindowsAsset)')"
+             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/win-x64/native/" />
+    <Content Include="@(NativeLinuxAsset)"
+             Condition="Exists('@(NativeLinuxAsset)')"
+             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/linux-x64/native/" />
+  </ItemGroup>
+
 </Project>
@@ -2,12 +2,14 @@ ARG IMAGE="sparkd"
 ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
 ARG SOURCE_DIR="install/sparkd-release-gcc"
 ARG PYTHON_REQUIREMENTS="requirements.txt"
+ARG SPARKD_ENTRYPOINT="src/app_sparkd/sparkd-entrypoint.sh"

 # sparkd_base
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_base
 ARG CONFIG_DIR
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS
+ARG SPARKD_ENTRYPOINT

 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -21,6 +23,7 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
 COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/.local/share/bts/sparkd/conf"
+COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"

 # sparkd
@@ -35,6 +38,7 @@ COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/.local/share/bts/sparkd/da
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_symbols
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS="requirements.txt"
+ARG SPARKD_ENTRYPOINT

 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -47,6 +51,7 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"

 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"

 # selected image
@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/all_crypto
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/gateio_dev/requirements.txt
+SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
@@ -0,0 +1,11 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "BTC",
+    "ETH",
+    "SOL",
+    "XRP"
+  ]
+}
@@ -0,0 +1,16 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "BTC",
+    "ETH",
+    "SOL",
+    "XRP"
+  ],
+  "use_testnet": false,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}
@@ -0,0 +1,71 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    # python_md_providers {
+    #   venue: VENUE_GATEIO
+    #   module_path: "python_md_providers"
+    #   provider_type: "gateio_md_provider.GateIoMdProvider"
+    # }
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_interop_plugins"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Gateio].symbol
+        }
+      }
+    }
+     okx {
+       num_depth_levels: 5
+       url_rest_base: "https://www.okx.com"
+       url_ws_base: "wss://ws.okx.com:8443"
+       security_criteria {
+         clauses {
+           security_types: SECURITY_TYPE_FUTURE
+           symbols: @SymbolLists.list[Gateio].symbol
+         }
+       }
+       is_application_ping_pong_enabled: true
+       application_ping_interval_ms: 20000
+       application_pong_timeout_ms: 10000
+       is_protocol_ping_pong_enabled: true
+       protocol_ping_interval_ms: 60000
+       protocol_pong_timeout_ms: 30000
+     }
+  }
+}
@@ -0,0 +1,32 @@
+  SymbolLists {
+  value {
+    list {
+      key: "Gateio"
+      symbol: "SOL"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GATEIO"
+      spot: @SymbolLists.list[Gateio].symbol
+    }
+    venues {
+      name: "VENUE_BYBIT"
+      future: @SymbolLists.list[Gateio].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[Gateio].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      future: @SymbolLists.list[Gateio].symbol
+    }
+  }
+}
@@ -0,0 +1,7 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_dev_replay_aws
+SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_dev_replay_aws/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
+SPARKD__SHM_SIZE=512m
+SPARKD__MEMORY=1g
@@ -1,18 +1,12 @@
-JetFIX {
+DeploymentType {
  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
+    type: "prod"
  }
 }
-
 PublisherOptions {
   value {
-      network_interface_a: 'ens3f0np0'
-      network_interface_b: 'ens3f1np1'
+      network_interface_a: 'ens5'
+      network_interface_b: 'ens5'
      #these get overriden if not entered here
      heartbeat_ms: 5000
      heartbeat_boot_seconds: 60
@@ -41,7 +35,7 @@ CMEOptions {
      use_tcp_replay: false
      recovery_queue_size: 640000
      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
      #max_gap_recovery_cache_size: 4
      use_natural_refresh: false
      join_snapshot_on_gap: false
@@ -73,32 +67,17 @@ MarketDataConfig {
       unique_check_value: 15
       use_databento: false
    }
-  }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
 }
-
-# JetFIX {
-#    value {
-#     #connection: @JetFIXSimConnection.sim
-#     connection {
-#     name: "mlx_004_54"
-#     venue: VENUE_GLBX
-#     socket_connect_host_ip: "69.50.112.191"
-#     socket_connect_port: 42876
-#     sender_compid: "MLX004"
-#     target_compid: "CME"
-#     target_subid: "54"
-#     cme_market_segment_id: 54
-#     start_time: "00:10:00"
-#     end_time: "23:55:00"
-#     start_day: "sunday"
-#     end_day: "friday"
-#     seconds_between_hbs: 30
-#     sender_subid: "BTS"
-#     use_ilink3: true
-#   #   raw_data: "S4PMA"
-#   #   auto_fix_seq_num: true
-#   }
-#    }
-# }
-
-
@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    # list {
+    #   key: "GlbxOptions"
+    #   symbol: "E1A"
+    #   symbol: "E1B"
+    #   symbol: "E1C"
+    #   symbol: "E1D"
+    #   symbol: "E2A"
+    #   symbol: "E2B"
+    #   symbol: "E2C"
+    #   symbol: "E2D"
+    #   symbol: "E3A"
+    #   symbol: "E3B"
+    #   symbol: "E3C"
+    #   symbol: "E3D"
+    #   symbol: "E4A"
+    #   symbol: "E4B"
+    #   symbol: "E4C"
+    #   symbol: "E4D"
+    #   symbol: "E5A"
+    #   symbol: "E5B"
+    #   symbol: "E5C"
+    #   symbol: "E5D"
+    #   symbol: "EW1"
+    #   symbol: "EW2"
+    #   symbol: "EW3"
+    #   symbol: "EW4"
+    #   symbol: "ESH9"
+    #   symbol: "ESZ6"
+    #   symbol: "ESH0"
+    #   symbol: "ESM6"
+    #   symbol: "ESZ7"
+    #   symbol: "ESH6"
+    #   symbol: "ESM7"
+    #   symbol: "ESZ8"
+    #   symbol: "ESH7"
+    #   symbol: "ESH8"
+    #   symbol: "ESU6"
+    #   symbol: "ESZ5"
+    #   symbol: "ESZ9"
+    #   symbol: "E1D"
+    #   symbol: "ESZ5"
+    #   symbol: "ES"
+    # }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        # option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}
@@ -0,0 +1,10 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
+SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g
+SPARKD__CONFIG_PORT=9000
+SPARKD__QUICKBUS_PORT=9001
+SPARKD__SIM_FIX_PORT=9002
@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1
@@ -0,0 +1,140 @@
+DeploymentType {
+  value {
+    type: "dev"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'ens65f1'
+      network_interface_b: 'ens65f0'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "maw_004_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "69.50.112.177"
+      socket_connect_port: 44040
+      sender_compid: "MAW004"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 2
+      max_connection_attempts: 100
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}
+Ports {
+   value {
+      config_port: 9000
+      qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
+      replicator_port: 9080
+   }
+}
@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
@@ -1,8 +0,0 @@
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GLBX"
-      future: "ES"
-    }
-  }
-}
@@ -0,0 +1,7 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
+SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g
@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1
@@ -0,0 +1,152 @@
+DeploymentType {
+  value {
+    type: "prod"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'enp193s0f0np0'
+      network_interface_b: 'enp193s0f1np1'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "u6p_7qb_54"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.97"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "54"
+      cme_market_segment_id: 54
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_order_msgs_per_window: 300
+      max_connection_attempts: 2
+      use_ilink3: true
+    }
+   connection {
+      name: "u6p_7qb_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.1"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_connection_attempts: 2
+      max_order_msgs_per_window: 300
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}
@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+      symbol: "ESH9"
+      symbol: "ESZ6"
+      symbol: "ESH0"
+      symbol: "ESM6"
+      symbol: "ESZ7"
+      symbol: "ESH6"
+      symbol: "ESM7"
+      symbol: "ESZ8"
+      symbol: "ESH7"
+      symbol: "ESH8"
+      symbol: "ESU6"
+      symbol: "ESZ5"
+      symbol: "ESZ9"
+      symbol: "E1D"
+      symbol: "ESZ5"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}
@@ -0,0 +1,7 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
+SPARKD__DATA_DIR=/home/dk/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/dk/8000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g
@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1
@@ -0,0 +1,101 @@
+DeploymentType {
+  value {
+    type: "prod"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'enp193s0f0np0'
+      network_interface_b: 'enp193s0f1np1'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+      symbol: "ESH9"
+      symbol: "ESZ6"
+      symbol: "ESH0"
+      symbol: "ESM6"
+      symbol: "ESZ7"
+      symbol: "ESH6"
+      symbol: "ESM7"
+      symbol: "ESZ8"
+      symbol: "ESH7"
+      symbol: "ESH8"
+      symbol: "ESU6"
+      symbol: "ESZ5"
+      symbol: "ESZ9"
+      symbol: "E1D"
+      symbol: "ESZ5"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}
@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
-SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf
@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay/requirements.txt
+TZ=America/Chicago
+SPARKD__MD_RECORDING_DIR=/home/wedbushprod/8000/md_recording
@@ -0,0 +1 @@
+requests==2.31.0
@@ -24,38 +24,9 @@ PublisherOptions {
  }
 }

-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Warn"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "OrderMan"
-        level: "Diag"
-      }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
-    }
-  }
-}
-
 MarketDataConfig {
  value {
-    replay {
+   replay {
      venue: VENUE_GLBX
    }
  }
@@ -0,0 +1,46 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "ES"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}
@@ -0,0 +1,9 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay_wb
+SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
+SPARKD__MD_RECORDING_DIR=/home/wedbushprod/md_recording
+SPARKD__CONFIG_PORT=9000
+SPARKD__QUICKBUS_PORT=9001
+SPARKD__SIM_FIX_PORT=9002
@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1
@@ -0,0 +1,46 @@
+JetFIX {
+  value {
+    connection {
+      name: "glbx_1"
+      venue: VENUE_GLBX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+Ports {
+   value {
+      config_port: 9000
+      qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
+   }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+     replay {
+      venue: VENUE_GLBX
+    }
+  }
+}
+
@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+      symbol: "ESH9"
+      symbol: "ESZ6"
+      symbol: "ESH0"
+      symbol: "ESM6"
+      symbol: "ESZ7"
+      symbol: "ESH6"
+      symbol: "ESM7"
+      symbol: "ESZ8"
+      symbol: "ESH7"
+      symbol: "ESH8"
+      symbol: "ESU6"
+      symbol: "ESZ5"
+      symbol: "ESZ9"
+      symbol: "E1D"
+      symbol: "ESZ5"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+init_timezone() {
+  tz="${1:-$TZ}"
+  zoneinfo_dir=/usr/share/zoneinfo
+  if [ -f "$zoneinfo_dir/$tz" -a "$(readlink -f /etc/localtime)" != "$zoneinfo_dir/$tz" ]; then
+    echo "Setting locales and timezone to $tz"
+    if [ -e /etc/localtime ]; then
+      sudo -n rm -f /etc/localtime
+    fi
+    sudo -n ln -snf "$zoneinfo_dir/$tz" /etc/localtime
+    echo $tz | sudo -n tee /etc/timezone > /dev/null
+  fi
+}
+
+load_env() {
+  f="$1"
+  if [ -e "$f" ]; then
+    echo "Sourcing $f"
+    set -a
+    . "$f"
+    set +a
+  fi
+}
+
+symlink() {
+  src="$1"
+  dst="$2"
+  if [ -L "$dst" ]; then
+    rm "$dst"
+  fi
+  ln -s "$src" "$dst"
+}
+
+roll_log() {
+  f="$1"
+  if [ -e "$f" ]; then
+    mv "$f" "${f}_$(date '+%Y%m%d-%H%M')"
+  fi
+}
+
+# Acquire sudo permission once for the script (needed for a few calls)
+echo bts | sudo -S -v
+data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
+sudo -n chown -R bts:bts "$data_root"
+load_env "$data_root/conf/.env.local"
+init_timezone "$TZ"
+roll_log "$data_root/data/logging.sparkd/log"
+symlink "$data_root/conf/plugins" /home/bts/bin/plugins
+
+if  [ -e "/md_recording" -a ! -e "/home/bts/.local/share/bts/sparkd/data/md_recording" ]; then
+  symlink "/md_recording" /home/bts/.local/share/bts/sparkd/data/md_recording
+fi
+
+bash -c "${@:-./sparkd run}"
@@ -17,7 +17,6 @@
 #include "spark/orderman/OrderServiceServer.h"
 #include "spark/orderman/events/TradeSummarySignalDispatcher.h"
 #include "spark/orderman/order_factory/DynamicOrderRegistry.h"
-#include "spark/os/os.h"
 #include "spark/os/spark_data_directory.h"
 #include "spark/persistence/PersistenceServiceServer.h"
 #include "spark/plugin/SparkServices.h"
@@ -32,6 +31,7 @@
 #include "sparkd/execution/execution_gateway_provider.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
+#include "sparkd/md/market_data_session.h"
 #include "sparkd/md/recording/MDRecorder.h"
 #include "sparkd/position/MarginServiceServer.h"
 #include "sparkd/position/PositionServiceServer.h"
@@ -46,6 +46,7 @@
 namespace bts::cmds {

 using bts::api::ServiceContext;
+using bts::md::MarketDataSession;
 using bts::md::MDClient;
 using bts::md::TradeSummarySignalDispatcher;
 using bts::metrics::MetricsContainer;
@@ -107,7 +108,7 @@ static void execute() {
   auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
   // start work loop in background
   py_plugin_man->start_work_thread();
-   md::cmds::Run::run(py_plugin_man);
+   MarketDataSession md_session{py_plugin_man};

   api::synthetic::SyntheticSecurityService synthetic_service(sc);
   bts::md::MDClient mdclient(
@@ -1,11 +1,11 @@
 #include "DumpSecurity.h"
 #include "spark/api/domain/SecurityType.h"
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/core/TableFormatter.h"
 #include "spark/domain_model/Security.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/domain_model/SecurityType.h"
 #include "spark/domain_model/Venue.h"
+#include "spark/format/table.h"
 #include "spark/md/Diag/ShmDumper.h"
 #include "spark/md/SecurityDB.h"
 #include "spark/md/SymbolLoader.h"
@@ -23,7 +23,8 @@ using bts::api::SecurityType;
 using bts::api::SecurityType_Name;
 using bts::api::Venue;
 using bts::api::venue_parse_smart;
-using bts::core::TableFormatter;
+using bts::format::format_table;
+using bts::format::detail::TableSchema;
 using bts::md::securitytype_of;
 using bts::md::SymbolLoader;
 using bts::md::db::load_metadata;
@@ -69,7 +70,7 @@ void print_securities(function<bool(const Security &)> predicate) {

 void print_db_securities(function<bool(const Security &)> predicate) {
   vector<Security> secs;
-   const TableFormatter::Schema<Security> schema{
+   const TableSchema<Security> schema{
       make_tuple(
           "id", 12, [](const Security &s) { return std::to_string(s.id()); }),
       make_tuple(
@@ -93,7 +94,7 @@ void print_db_securities(function<bool(const Security &)> predicate) {
      secs.push_back(sec);
   };
   load_securities([](auto) { return true; }, on_sec);
-   TableFormatter::format(std::cout, secs, schema);
+   format_table(std::cout, secs, schema);
 }

 template <class T, class ParseT = function<bool(const string &, T *)>>
@@ -109,7 +109,8 @@ struct ExecutionGatewayProvider::PImpl {
         return make_gw<ICEExecutionGateway>(conn, conn, *l, o);
      case api::VENUE_GLBX:
         if (conn.use_ilink3()) {
-            make_gw<ilink3::GLBXiLink3ExecutionGateway>(conn, conn, *l, o);
+            return make_gw<ilink3::GLBXiLink3ExecutionGateway>(
+                conn, conn, *l, o);
         }
         return make_gw<GLBXExecutionGateway>(conn, conn, *l, o);
      case api::VENUE_DERIBIT:
@@ -23,7 +23,7 @@
 #include "spark/api/services/native_order_type.h"
 #include "spark/api/services/time_in_force.h"
 #include "spark/config/protos/jetfix.pb.h"
-#include "spark/core/TimeStamp.h"
+#include "spark/core/time_stamp.h"
 #include "spark/core/start_stop_state_machine.h"
 #include "spark/domain/tick_utils.h"
 #include "spark/domain_model/Security.h"
@@ -4,7 +4,7 @@
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/core/FMT.h"
-#include "spark/core/TimeSpan.h"
+#include "spark/core/time_span.h"
 #include "spark/domain/DomainSecurityUtils.h"
 #include "spark/domain/SecurityID.h"
 #include "spark/domain_model/Venue.h"
@@ -141,5 +141,6 @@ MarketDataGateway::MarketDataGateway(
 MarketDataGateway::~MarketDataGateway() = default;

 void MarketDataGateway::start() { me_->start(); }
+void MarketDataGateway::stop() { me_->stop(); }

 } // namespace bts::md
Author	SHA1	Message	Date
arham	1f94fe65bc	added more detail missing security log in gmdl	2026-10-26 03:32:49 +00:00
arham	edf8adb90e	config: new config for ophirs new sim server	2026-10-21 13:09:14 +00:00
arham	a4ff7743da	Add: simple doc to expplain security handling in sparkd	2026-09-04 13:28:03 +00:00
arham	2853af9e0f	add: locks to gmdl to handle multi-threaded producers and add a way to handled deffered spreads	2026-09-04 13:27:12 +00:00
arham	5482278b5b	ADD: MDP3 pipeline now points to gmdl in case of sime execution	2026-09-04 13:23:25 +00:00
arham	9732d90683	cme md recording and replay no loop	2026-09-01 05:28:30 +00:00
arham	53ddf9a173	cme raw packet recorder	2026-08-28 07:03:16 +00:00
arham	38d87cddcf	fix: white backgroud in datepicker in mf	2026-08-23 22:05:31 -04:00
arham	4d4b2cf967	add: date change for manual fills	2026-08-24 00:32:20 +00:00
John Colvin	eadd6a0706	fix: Update title in new ladder ticket when sec received	2026-08-14 14:50:31 -04:00
John Colvin	503a5e2715	fix: Revert ladder to use fixed price list instead of virtual rows	2026-08-14 14:42:12 -04:00
John Colvin	786736b38f	Add Orders column placeholder to new ladder grid	2026-08-14 14:42:12 -04:00
John Colvin	1ef2ed9d57	Make new ladder grid use 1 row per tick with fixed row count	2026-08-14 14:42:12 -04:00
John Colvin	b39ea9528c	Use fixed price rows (per tick) in new ladder	2026-08-14 14:42:12 -04:00
John Colvin	c59ce773a7	Set security in OnSecurityMd callback in new ladder	2026-08-07 17:20:14 -04:00
arham	400981738f	fix: bug fix now sell works	2026-08-05 15:47:21 -04:00
arham	3fd749dcd9	add: change colors and add ladder link	2026-08-05 14:04:32 -04:00
John Colvin	a250967286	feat: Launch modify ticket w/ ladder	2026-07-30 14:13:35 -04:00
John Colvin	a81126d888	feat: Launch ticket w/ ladder popup from mview	2026-07-30 14:13:35 -04:00
John Colvin	c38ff5926e	feat: Add new order ticket with ladder price levels	2026-07-30 14:13:35 -04:00
John Colvin	49df45574f	Rename get_ladder_profile PascalCase	2026-07-30 11:51:37 -04:00
John Colvin	7b2e19f7c8	Remove unused ext fn subscribe_ladder_profile	2026-07-30 11:51:37 -04:00
John Colvin	c9e3cf8c67	Rename AuxiliaryDataResources -> OrderAuxiliaryPopupResources The old name was unnecessarily vague	2026-07-30 11:48:00 -04:00
John Colvin	6b31f7383c	Remove unused code ConstrainedTextBlock	2026-07-30 11:48:00 -04:00
arham	f2c95d6c28	sec filter on the basis of expiration in mdp3	2026-07-27 09:32:10 +00:00
Taha	1d3bbcc75f	TC2 ATM highlighting wokrs now without AW2 setup	2026-07-22 16:52:49 -04:00
arham	94bfdacf8b	add: trade latency logs	2026-07-02 16:21:08 +00:00
arham	2a8d407f08	conf: add config to run crypto on aws, bybi and gate not working	2026-06-24 01:03:49 +00:00
Taha	55ed962c63	Fixing empty TC2 issue, caused by exceptions in _override_theos	2026-06-17 23:10:57 -04:00
arham	68f766e00b	fix: param syntax change for args	2026-06-17 10:33:29 -04:00
arham	c0252e06c5	fix: fill time nanos stored	2026-06-16 09:58:03 +00:00
John Colvin	4258bd4c82	fix: Use unique ProtoMember ids in orderbook FilterStates The Tag property had the same id as accounts (5)	2026-06-15 15:29:07 -04:00
John Colvin	4981d70bf2	fix: Return NaN instead of throwing if priceType invalid It's possible this fixes the ladder freezing bug reported by Ophir	2026-06-15 15:22:28 -04:00
John Colvin	b85c249a7d	improve: Map nullable Color to Color type in BTSColorExtension So we don't need "Type={x:Type Color}" in ColorAnimation xaml	2026-06-15 15:21:31 -04:00
John Colvin	6721e07cb6	fix: Use BTSColor markup ext instead of StaticResource	2026-06-15 15:08:28 -04:00
arham	8aa5529218	conf: increased shm mem size	2026-06-02 03:04:55 +00:00
arham	ba5fff0168	add: ping service to sdk	2026-06-01 10:28:06 +00:00
arham	976119fbee	fix: build was broken, spark.dll being used by another process	2026-06-01 04:31:57 -04:00
arham	ed741c0621	fix: sdk computes position from fills instead of using postion service	2026-06-01 08:02:44 +00:00
arham	e0cf3c739f	Add: backend il3gw fills aggressor field	2026-06-01 07:51:14 +00:00
arham	59e2519a4d	fix: fills aggressor flag displayed	2026-06-01 02:12:16 -04:00
arham	15de6f3e3d	add: added entries for weekly options cme to defualt cpp conf	2026-05-21 09:48:34 +00:00
John Colvin	2e36f00382	fix: Only set order tag if non-null	2026-05-14 10:43:01 -04:00
John Colvin	9f24a3e7a2	improve: Make sdk demo venue parse case insensitive	2026-05-14 10:43:01 -04:00
John Colvin	e8dbf6fe09	vscode: Set Sdk Demo prelaunch task to Build Spark Sdk	2026-05-14 10:43:01 -04:00
John Colvin	aa403b83ef	improve: Use quantity sign to indicate side in TopOfBookQuoter example	2026-05-14 10:43:01 -04:00
arham	c507f20a8e	fix: build broken for gcc	2026-05-13 12:33:57 +00:00
arham	ea52a4749d	conf: update conf	2026-05-13 12:33:44 +00:00
arham	650710b5e2	remove stale config	2026-05-12 11:53:31 +00:00
arham	20f6a29e05	add: conf for cert and prodexit	2026-05-12 11:51:25 +00:00
John Colvin	e2f8d4db11	fix build (deploy): Actually include native assets in nupkgs Native assets (libspark.dll/so) are copied to the outdir for local builds, and packed in nupkg files for sdk releases. The dest path in the package is different from the path in the outdir, so we configured Copy/Pack as separate msbuild items. However, both had the same Include="@(NativeWindows/LinuxAsset)", so Pack="false" in the Copy item was overriding Pack="true" in the Pack item.	2026-05-08 14:15:54 -04:00
arham	1812434ddc	Fix: front end exe polling decreased from 100ms to 1ms	2026-05-08 15:24:26 +00:00
Taha	fd50050064	Position cols in TC2 work again now	2026-05-07 03:57:07 +00:00
arham	6ac43881ae	add: add missing tag to default conf	2026-05-07 00:22:28 +00:00
arham	4af1419748	Add: timezone change to chi and std higher ram for multicast compose	2026-05-07 00:22:03 +00:00
arham	7cfcdc3770	fix: ilink3 code generator produces correct code that handles null values	2026-05-07 00:21:10 +00:00
arham	d03a705077	fix: ilink3 genereated files updated to handle null values with null instead of ascii zero	2026-05-07 00:20:34 +00:00
arham	58eccad70d	fix: fills ui price was zero now it works	2026-05-07 00:19:44 +00:00
arham	5f8ec04ff1	Add: useful tools to prod docker image	2026-05-07 00:19:16 +00:00
arham	05e2107655	Fix: change object pool to detect exe report memory leak from assert to error	2026-05-07 00:18:45 +00:00
arham	d41f613e32	FIx: change object pool limit for il3gw for exe reports	2026-05-07 00:17:52 +00:00
arham	bcbb598423	Add: better logs and disconnect handling	2026-05-04 20:42:23 +00:00
arham	e88c44f9db	CME TEMPLATE 9	2026-05-04 16:39:48 -04:00
arham	93625957d1	Fix: CRITICAL CME FIX	2026-04-29 03:58:45 +00:00
John Colvin	8eccb869e1	Remove unnecessary nuget stuff from externals	2026-04-08 00:13:01 -04:00
John Colvin	40f4333fc7	fix: SingleSecurityCriteria must have single id or venue/name pair	2026-04-07 17:23:25 +00:00
John Colvin	319d5c2384	vscode: Fix PubSubServer config environment param	2026-04-07 17:23:25 +00:00
arham	4a74713369	ophir multicast config replay and live working	2026-04-02 00:17:46 +00:00
arham	082beef1da	Rename	2026-03-31 23:59:04 +00:00
arham	f92d789e90	Add: recording config	2026-03-31 14:16:55 +00:00
arham	cd3e845dd9	Fix: set log to diag, it logs every book update	2026-03-31 14:09:37 +00:00
arham	a5b704eef1	Add: config for ophir wb	2026-03-31 14:09:05 +00:00
arham	afee63e061	Add: new docker compose to support mulitcast	2026-03-31 14:08:42 +00:00
John Colvin	fc9356d1a0	deploy: Optionally mount sparkd entrypoint in config dir To mount a custom entrypoint script, put it in the config profile dir and define the following env vars: SPARKD__IMAGE=sparkd_symbols SPARKD__MAIN_ENTRYPOINT=/home/bts/.local/share/bts/sparkd/conf/<custom-entrypoint> To invoke the "normal" sparkd entrypoint script from the mounted one: ./sparkd-entrypoint.sh	2026-03-27 18:28:25 +00:00
John Colvin	a85f22c6a8	deploy: Fix sudo issues in sparkd-entrypoint.sh	2026-03-27 17:59:20 +00:00
John Colvin	2024eba4d0	deploy: Add ophir_sim_replay config profile for sparkd	2026-03-26 16:20:41 -04:00
John Colvin	1a49a828ad	build: Don't activate venv when generating individual protos We activate the venv for the whole build	2026-03-26 16:20:41 -04:00
John Colvin	c0448c7c8d	deploy: Resolve profile dirs with regex patterns	2026-03-26 16:20:41 -04:00
John Colvin	a6c6e7faf3	improve: Wait until py work requested before loading py modules So we don't crash due to missing py modules when no py plugins are running.	2026-03-26 16:20:41 -04:00
John Colvin	56445d2d66	deploy: Add sparkd-shell service to compose.yaml for troubleshooting	2026-03-26 16:20:41 -04:00
John Colvin	ccf1bd51c2	cleanup: Remove unused code	2026-03-26 16:20:41 -04:00
John Colvin	fd21269df7	refactor: Wrap md run logic in class MarketDataSession Start in the constructor, gracefully exit in the destructor.	2026-03-26 16:20:41 -04:00
John Colvin	fe85c3e648	Allow md recording dir to be mounted as volume in docker compose	2026-03-26 16:20:41 -04:00
John Colvin	e1d33ecadf	deploy: Configure timezone in entrypoint script	2026-03-26 16:20:41 -04:00
John Colvin	83df96c70f	deploy: Move compose entrypoint to dedicated script	2026-03-26 16:20:41 -04:00
John Colvin	f718f5b15c	deploy: Fix typo in sparkd conf dir path in compose.yaml	2026-03-26 16:20:41 -04:00
John Colvin	5d752f3b73	refactor: Replace DataPacket with IDataPacket in more places	2026-03-26 16:20:40 -04:00
John Colvin	d308dbf15a	build: Check if venv already active before activating	2026-03-26 16:20:40 -04:00
John Colvin	21c43b9240	refactor: Replace TableFormatter with table.h Also move to "format" dir	2026-03-26 16:20:40 -04:00
John Colvin	e1553f1b04	refactor: Replace StringFinder/Joiner.h with find/join.h Also move to "algorithm" dir	2026-03-26 16:20:40 -04:00
John Colvin	a26918efe9	refactor: Rename a few TitleCase .h files snake_case * EnumMacros.h -> enum_macros.h * TimeSpan.h ---> time_span.h * TimeStamp.h --> time_stamp.h	2026-03-26 16:20:40 -04:00
John Colvin	f729fc4c59	diag: Use structured logs in MDP3Listener::onOption	2026-03-26 16:20:40 -04:00
John Colvin	fa67adbe36	deploy: Install tzdata in linux_dotnet8 Dockerfile So we can configure timezone in compose services. Specifically, we need this to support live cme md (so onixs doesn't think messages are "too old").	2026-03-26 16:20:40 -04:00
John Colvin	72f9ec582e	cleanup: Remove unused code	2026-03-26 16:20:40 -04:00
John Colvin	8fea31bdf6	refactor: Add new glbx variable ticks to security (shm and proto)	2026-03-26 16:20:39 -04:00
John Colvin	ca96e41391	build: Upgrade cs LangVersion 11 -> 12	2026-03-26 16:20:39 -04:00
John Colvin	d4e5c95fde	refactor: Rename SparkApiCredentials -> SparkCredentials	2026-03-26 16:20:39 -04:00
John Colvin	f6e0ec3617	refactor: Rename SparkdLoginService -> SparkLoginService This makes it consistent with other type names (we don't use "Sparkd" anywhere else).	2026-03-26 16:20:39 -04:00
John Colvin	3fa5284838	refactor: Rename SparkdCredentials.cs -> SparkApiCredentials.cs Matches the class name	2026-03-26 16:20:39 -04:00
John Colvin	4dcaa565f6	cleanup: Remove unnecessary subscription wrapper in cs sdk	2026-03-26 16:20:39 -04:00
John Colvin	75ecfa89d6	build: Clean up spark_client.sln * Remove unused csprojs related to old sdk * Remove erroneous build type config	2026-03-26 16:20:39 -04:00
John Colvin	1c930439a4	fix build (dsqa): Add using Spark.Api.MarketData Needed to reference IMarketDataClient	2026-03-26 16:19:30 -04:00
John Colvin	25084b8cd9	fix build (cs): Add "global::" prefix when using Spark.Api ns Resolves conflicts with the Bts.Spark.Api namespace when the using statement is nested inside the root "Bts" namespace.	2026-03-26 16:17:16 -04:00
John Colvin	5b8a8a8882	build: Simplify target to include native assets in cs	2026-03-08 16:00:49 +00:00
John Colvin	21052090a2	fix (pre-emptive): Format dotnet flag opts correctly In pack_spark_sdk.py, the dotnet pack cmd is constructed with a dict of opts. If the value part is None, the opt should be formatted as a flag, "--key" instead of "--key None".	2026-03-07 13:32:03 -05:00
John Colvin	9bd32b8225	fix build (pack cs sdk): Fix --(no-)cs toggle behavior * Always run "dotnet pack", regardless of csharp toggle opt * Run "dotnet publish" if csharp toggled on	2026-03-07 13:20:56 -05:00
John Colvin	e8e234f420	fix: Don't fire empty updates in Books IObservable	2026-03-07 01:19:38 -05:00
John Colvin	be23237a45	fix: Update last val in IObservable FlattenSeries ext The bootstrap delegate that stitches the sequences together wasn't getting properly invoked because of this.	2026-03-07 01:19:28 -05:00