build: Install boost via vcpkg on linux, update ccapi

Old ccapi wasn't compatible with new boost
build: Compile spark-windows with clang
2026-03-20 17:02:44 +00:00 · 2026-03-20 17:02:44 +00:00 · 2026-03-20 17:02:44 +00:00 · 2026-03-20 17:02:44 +00:00 · 2026-03-20 17:02:44 +00:00 · 2026-03-20 17:02:44 +00:00
526 changed files with 4262 additions and 25092 deletions
@@ -51,8 +51,7 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
 SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
@@ -183,10 +183,11 @@
      "windows": {
        "program": "PubSubServer.exe",
      },
      "args": [
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
      ],
      "cwd": "build/Debug/bin/net8.0",
      "env": {
        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -213,7 +214,7 @@
      "type": "coreclr",
      "request": "launch",
      "program": "",
-      "preLaunchTask": "Build Spark Sdk (Debug)",
+      "preLaunchTask": "Build Api Server (Debug)",
      "windows": {
        "program": "Bts.Spark.Sdk.Demo.exe",
      },
@@ -285,7 +286,7 @@
      "windows": {
        "program": "Spark.exe",
        "args": [
-          "--Spark:ConfigServer",
+          "--sparkd",
          "${input:sparkConfigServer}"
        ],
        "cwd": "build/Debug/bin/net8.0-windows",
@@ -448,15 +449,14 @@
      "default": "Local",
    },
    {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
      "type": "pickString",
-      "description": "PubSubServer Configuration",
+      "description": "PubSub Configuration",
      "options": [
-        "Development",
+        "PubSubServer",
-        "EchoServer",
+        "SimpleEchoClient",
        "Local",
      ],
-      "default": "Development",
+      "default": "PubSubServer",
    },
    {
      "id": "sdkDemoEnvironment",
@@ -17,6 +17,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{3BF4DF96-5AFF-4674-866A-71A8090627FF}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{62067659-AA2E-4CB5-B7CE-0B38EAA92080}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -87,6 +95,54 @@ Global
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.Build.0 = Debug|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.Build.0 = Debug|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.Build.0 = Release|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.ActiveCfg = Release|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.Build.0 = Release|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.Build.0 = Release|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.Build.0 = Debug|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.Build.0 = Debug|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.ActiveCfg = Release|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.Build.0 = Release|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.ActiveCfg = Release|Any CPU
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.Build.0 = Release|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.Build.0 = Debug|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.Build.0 = Debug|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.Build.0 = Release|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.ActiveCfg = Release|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.Build.0 = Release|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.ActiveCfg = Release|Any CPU
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.Build.0 = Release|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.Build.0 = Debug|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.Build.0 = Debug|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.Build.0 = Release|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.ActiveCfg = Release|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.Build.0 = Release|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.ActiveCfg = Release|Any CPU
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -98,5 +154,9 @@ Global
 		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{3BF4DF96-5AFF-4674-866A-71A8090627FF} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{62067659-AA2E-4CB5-B7CE-0B38EAA92080} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal
@@ -32,14 +32,20 @@
        "lhs": "${hostSystemName}",
        "rhs": "Windows"
      },
-      "generator": "Visual Studio 17 2022",
+      "generator": "Ninja Multi-Config",
      "environment": {
-        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
+        "SPARK_CMAKE_CXX_FLAGS": "/WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang-cl",
        "CMAKE_CXX_COMPILER": "clang-cl",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
        "CMAKE_EXPORT_COMPILE_COMMANDS": {
          "type": "BOOL",
          "value": "ON"
        },
        "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
@@ -51,6 +51,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{802ECF0B-7264-4168-95D1-158E47FB65CC}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{0354655D-2394-42BB-B1D4-2D75B9BCABF1}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -571,6 +579,102 @@ Global
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.ActiveCfg = Release|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.Build.0 = Release|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.ActiveCfg = Release|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.Build.0 = Release|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.ActiveCfg = Release|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.Build.0 = Release|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.Build.0 = Release|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.ActiveCfg = Release|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.Build.0 = Release|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.ActiveCfg = Release|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.Build.0 = Release|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.Build.0 = Release|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.ActiveCfg = Release|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.Build.0 = Release|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.ActiveCfg = Release|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.Build.0 = Release|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.ActiveCfg = Release|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.Build.0 = Release|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -592,6 +696,10 @@ Global
 		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{802ECF0B-7264-4168-95D1-158E47FB65CC} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{0354655D-2394-42BB-B1D4-2D75B9BCABF1} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
@@ -10,7 +10,7 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+        - ENTRYPOINT_SCRIPT=${SPARKD__ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    env_file:
      - .env
@@ -19,7 +19,6 @@ services:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      resources:
        limits:
@@ -36,7 +35,7 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+        - ENTRYPOINT_SCRIPT=${SPARKD__ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: "do sleep 1; done"
    env_file:
@@ -54,7 +53,6 @@ services:
  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
@@ -63,7 +61,7 @@ services:
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+        - ENTRYPOINT_SCRIPT=${SPARKD__ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    env_file:
      - .env
@@ -72,7 +70,6 @@ services:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
@@ -80,65 +77,7 @@ services:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
-  sparkd-multicast:
+
    environment:
      - TZ=America/Chicago
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    network_mode: "host"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-16gb}"
  sparkd-multicast-cert:
    environment:
      - TZ=America/Chicago
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    network_mode: "host"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-16gb}"
  sparkd-tail:
    image: bluetradingsystems/linux_dotnet8
    volumes:
@@ -1,942 +0,0 @@
 # Security ID handling in `sparkd`
 This document traces how security identifiers are created, translated, persisted,
 indexed, and published by the `sparkd` market-data paths. It focuses on the
 legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
 ## Executive summary
 `sparkd` uses two identifiers that must not be confused:
 | Identifier | Meaning | Typical field | Lifetime |
 |---|---|---|---|
 | Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
 | Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
 `invalid_security_key` is `0`; a security with key `0` is not yet publicly
 discoverable through the shared-memory consumer path.
 The two principal paths differ:
 - **MDP3/OnixS:** the definition callback sets only the CME ID in
  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
  calls `MDProducerHandle::set_key()`.
 - **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
  currently uses the ID already present on the incoming domain security
  (`use_given_sec_key == true`). Live adapters normally obtain that ID
  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
  it to both `FixedSecurity::key` and `numeric_id`, so
  `MDProducer::add_security()` publishes it synchronously. Only the listener's
  own fallback `IDGenerator` call is bypassed by the current constant.
 ## End-to-end diagrams
 ### CME MDP3 / OnixS
 ```mermaid
 sequenceDiagram
    participant FE as OnixS feed-engine thread
    participant L as MDP3Listener
    participant P as MDProducer / shared memory
    participant Q as SecurityIDRecorder queues
    participant D as sparkd dispatcher thread
    participant DB as key_ids.sqlite
    FE->>L: instrument definition (CME SecurityID)
    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
    L->>P: add_security(FixedSecurity)
    P->>P: allocate hot + cold entries with key 0
    P->>Q: ensure_key(handle)
    L->>L: index[CME SecurityID] = handle
    Note over FE,L: Books can route by CME ID while the Spark key is pending
    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
    Q->>DB: find (namespace, descriptor), or insert a new row
    DB-->>Q: SQLite rowid = Spark security key
    Q->>P: handle.set_key(rowid)
    P->>P: hot.key first, then cold security.key, then increment security_count
    Note over P: The security is now public to shared-memory consumers
 ```
 ### Generalized/newer pipeline (current behavior)
 ```mermaid
 sequenceDiagram
    participant A as Venue adapter / plugin
    participant DB as IDGenerator / key_ids.sqlite
    participant G as GeneralMarketDataListener
    participant I as GeneralSecurityIndex
    participant P as MDProducer / shared memory
    A->>DB: get_or_add(venue, type, adapter key)
    DB-->>A: SQLite rowid as domain SecurityID
    A->>G: on_securities(domain Security list with ID)
    G->>G: _find_sec() - use incoming security_common.id
    G->>G: _copy_security_common() - key = id, numeric_id = id
    G->>P: add_security(FixedSecurity with non-zero key)
    P->>P: set_key() during add_security()
    G->>I: index[id] = handle
    Note over G,P: No asynchronous ID assignment in the current configuration
 ```
 ## Data-flow charts
 The sequence diagrams above emphasize timing and thread transitions. These
 flowcharts emphasize which identifier is carried through each representation
 and where data is retained.
 ### MDP3 / legacy data flow
 ```mermaid
 flowchart TD
    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
    ONIXS --> DEF["MDP3Listener definition callback"]
    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
    FIXED --> PRODUCER["MDProducer::add_security"]
    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
    UNRESOLVED --> CMEINDEX
    CMEINDEX --> LEGS
    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
    INSERT --> ROWID
    ROWID --> SETKEY["MDProducerHandle::set_key"]
    SETKEY --> HOT["Write hot-entry key"]
    HOT --> COLD["Write cold security key"]
    COLD --> COUNT["Increment security_count"]
    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
    UPDATE --> CMEINDEX
    CMEINDEX --> HANDLE["MDProducerHandle"]
    HANDLE --> LIVE["Update live shared-memory data"]
    PUBLIC --> LIVE
 ```
 The CME index is usable before the Spark key is assigned, so feed updates can
 reach the internal handle while the security is still hidden from consumers.
 The persistent rowid crosses the publication boundary only at `set_key()`.
 ### Generalized/newer data flow
 ```mermaid
 flowchart TD
    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
    IDINPUT --> MUTEX["IDGenerator global mutex"]
    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
    SELECT -- Existing row --> ROWID["SQLite rowid"]
    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
    INSERT --> ROWID
    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
    IDTYPE -- Recorded or pre-keyed --> DOMAINID
    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
    CONVERT --> PRODUCER["MDProducer::add_security"]
    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
    PYHOT --> EVENTID
    PYHOT -. uses same mutex and SQLite path .-> MUTEX
    EVENTID --> CCMAP
    CCMAP --> GENERALINDEX
    GENERALINDEX --> HANDLE["MDProducerHandle"]
    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
    SHM --> SHMUPDATE
 ```
 In the normal newer live path, persistence occurs before the definition enters
 the generalized listener. The DB call therefore adds synchronous adapter-side
 latency, but publication needs no later timer. Most adapters use an in-memory
 map for subsequent events; the dotted Python path is the documented exception.
 ## The publication boundary
 `MDProducerHandle::set_key()` is the definitive publication point for the
 legacy shared-memory representation. Its write ordering is intentional:
 1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
 2. Write the key into the cold security entry.
 3. Increment the cold header's public `security_count`.
 The cold security key is written last because consumers treat a non-zero cold
 key as visibility. At that moment the corresponding hot entry must already be
 addressable. `set_key()` only permits the transition from `0` to one key; it is
 not a key-update mechanism.
 ## Identifier representations
 The names are historically inconsistent. In particular, many protobuf fields
 named `security_id` contain the **Spark key**, not the exchange's ID.
 | Layer | Exchange/feed identifier | Spark identifier |
 |---|---|---|
 | MDP3 wire/OnixS | `securityId()` | not present |
 | Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
 | Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
 | New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
 | Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
 | Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
 | CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
 | General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
 The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
 shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
 64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
 allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
 no explicit checked conversion.
 ## Which parts use exchange IDs, descriptors, or Spark security IDs?
 The three identifier classes serve different jobs:
 - **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
 - **Descriptor/natural key:** recognizes a definition logically at the
  persistence, discovery, or human-facing boundary.
 - **Spark security ID/key:** joins the instrument to books, orders, fills,
  positions, and services after publication.
 The core downstream system does not repeatedly identify a security from its
 description. The description is primarily used to obtain a Spark key; after
 that boundary, most joins use the numeric Spark key.
 ### Component-level usage matrix
 | Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
 |---|---|---|---|
 | CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
 | `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
 | MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
 | MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
 | Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
 | `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
 | Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
 | Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
 | Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
 | `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
 | `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
 | `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
 | `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
 | `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
 | `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
 | MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
 | Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
 | Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
 | Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
 | Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
 ### Exchange/external IDs: where they stop
 In MDP3 the separation is explicit:
 ```text
 CME SecurityID
    -> FixedSecurity.numeric_id
    -> CMESecurityIndex lookup
    -> MDProducerHandle
    -> Spark security_key for downstream publication
 ```
 The external CME ID continues to route feed messages but does not become the
 order/position key. This permits the same downstream Spark key to survive a
 restart as long as the logical descriptor mapping remains in
 `key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
 The generalized path collapses more of these roles:
 ```text
 exchange/vendor identity
    -> adapter calls IDGenerator
    -> domain::SecurityID
    -> FixedSecurity.key and FixedSecurity.numeric_id
    -> GeneralSecurityIndex and downstream Spark key
 ```
 `SecurityCommon::exchange_sec_id` may still contain the original textual
 external identifier, but `GeneralMarketDataListener` does not use it to route
 events and does not put it in `numeric_id`. The adapter must translate later
 events to `domain::SecurityID` before invoking the listener.
 ### Descriptor/name users
 Descriptors are used in three main places:
 1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
   descriptor, while `IDGenerator` uses its caller-provided `name`.
 2. **Definition lookup and discovery:** `SecurityCache` supports
   venue/type/name lookup; symbol filters and administrative tools use security
   metadata to select or display instruments.
 3. **Diagnostics:** logs include symbols and descriptions so humans can
   understand a numeric key.
 Descriptions generally do not drive the high-rate book, trade, order, fill, or
 position paths. Those paths use numeric IDs after definition resolution.
 It is important that “descriptor” does not mean one universal canonical field:
 - Legacy CME uses Globex `security_desc` from tag 55.
 - Many generalized adapters use `sec.desc()` or an exchange instrument name.
 - Databento uses its numeric instrument ID converted to a string.
 - Python uses `exchange_security_id`.
 Therefore, two adapters may describe the same economic contract but request
 different logical rows and receive different Spark IDs.
 ### Spark security-ID users
 Once a security is published, its Spark key is the dominant system identifier:
 ```mermaid
 flowchart LR
    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
    KEY --> MDC["MDConsumer and book subscriptions"]
    KEY --> MBO["MBO snapshots and queries"]
    KEY --> OMS["Orders, algorithms, and spread legs"]
    KEY --> FILLS["Fills and execution records"]
    KEY --> POS["Positions, PNL, and risk"]
    KEY --> API["Services and client APIs"]
    KEY --> REC["Relay and market-data recordings"]
    KEY --> SECDB["Serialized security definitions"]
 ```
 Fields named `security_id`, `security_key`, or top-level `Security.id()` in
 these downstream areas almost always mean the Spark key. They should not be
 compared directly with CME `SecurityID`, Databento `instrument_id`, or another
 venue/vendor identifier without going through the relevant adapter/index
 mapping.
 ## Persistence and allocation
 ### The authoritative key mapping: `key_ids.sqlite`
 Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
 file is:
 ```text
 <SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
 ```
 The database contains one table and two indexes:
 ```sql
 CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
 CREATE UNIQUE INDEX key_index ON store(ns, key);
 CREATE INDEX ns_index ON store(ns);
 ```
 The SQLite `rowid` is the generated Spark key. `ns` is normally
 `<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
 descriptor chosen by the caller. `value` is normally empty. Therefore ID
 stability depends on all three of these remaining stable:
 - the configured `data_path` and preservation of this database;
 - the venue/type namespace;
 - the descriptor supplied by the feed adapter.
 The two allocator implementations share the file and convention:
 - `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
  uses `securities::descriptor()`. For CME this is the Globex
  `security_desc`, populated from the definition's symbol/tag 55.
 - `IDGenerator` (newer): derives the namespace from new domain enums and uses
  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
  identifier; Databento deliberately passes the decimal instrument ID, and the
  Python bridge uses the exchange security ID.
 ### Logical-row examples
 The following names are illustrative. Production uses the exact string emitted
 by the selected feed/adapter, including its case, spacing, and weekly-series
 notation.
 #### Legacy `SecurityIDRecorder`
 For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
 Globex `security_desc`. After converting the fixed security to
 `api::Security`, `securities::descriptor()` returns that same
 `glbx.security_desc()` string.
 | Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
 |---|---|---|---|
 | Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
 | Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
 | Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
 | Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
 A weekly option does not get a special namespace. Standard and weekly options
 are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
 `security_desc` distinguishes the weekly series. For example, if the database
 assigned rowids `42001` through `42004` to the rows above, those integers—not
 the descriptor strings—would be the Spark security keys.
 Conceptually, legacy allocation is:
 ```text
 future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
 option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
 weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
 spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
 ```
 #### Newer `IDGenerator`
 The namespace result is similar, but the third argument is chosen by the
 adapter. Examples using a description-oriented adapter are:
 ```cpp
 get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
 // store: ns="GLBX.FUTURE", key="ESZ6"
 get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
 // store: ns="GLBX.OPTION", key="ESZ6 C6500"
 get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
 // store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
 get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
 // store: ns="COINBASE.SPOT", key="BTC-USD"
 ```
 Databento intentionally produces a different logical row even when the human
 description is `ESZ6`. If its instrument ID were `12345678`, it would call the
 equivalent of:
 ```cpp
 get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
 // store: ns="GLBX.FUTURE", key="12345678"
 ```
 Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
 `exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
 while a provider that sends `exchange_security_id="BTCUSD"` produces a
 different row and therefore a different Spark key.
 These examples show why the tuple is only logical when the adapter's naming
 contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
 describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
 Both enum-prefix removals currently produce the same namespace spelling (for
 example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
 rowid space. Live rows cannot share a rowid. The table does not declare
 `AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
 SQLite to reuse a number later. Security-key rows should therefore be treated
 as append-only operational data.
 `SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
 and feed latency over maximum durability against a host/power failure. The
 unique `(ns, key)` index prevents two different rows for the same logical
 security.
 ### Other places the key is stored
 | Location | Purpose | Authoritative for allocation? |
 |---|---|---|
 | Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
 | `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
 | `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
 | Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
 | Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
 The `SecurityIDRecorder::work_db()` comment says a published security is also
 recorded in the security database, but that function does **not** write
 `securities.sqlite` in the current code. Full security definitions are persisted
 separately by `SecurityCache::flush_to_disk()` through
 `db::insert_or_replace()`.
 ## CME MDP3 / OnixS pipeline in detail
 ### 1. Construction and thread setup
 `MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
 `MDProducer`. It constructs the recorder with:
 - a lookup callback that maps `ExchangeKey` through the CME index to the
  handle's current Spark key; and
 - `Dispatcher::current()`, which drives a 200 ms recorder timer.
 In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
 dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
 then calls that thread's dispatcher `run()`. The older `md run onixs` command
 uses its command/dispatcher thread similarly.
 The OnixS feed engine has its own configured thread pool. Instrument and market
 data callbacks therefore originate on OnixS feed-engine threads, while recorder
 database work and final key assignment execute on the MDP3 dispatcher thread.
 | MDP3 stage | Executing thread | DB call? |
 |---|---|---|
 | Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
 | Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
 | Add CME-ID index entry | Same OnixS callback thread | No |
 | Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
 | Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
 | `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
 ### 2. Definition callback: exchange ID is set, Spark key is not
 For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
 ```cpp
 fsec.numeric_id = def.securityId();
 ```
 It also copies symbol/descriptor/venue and pricing fields. A newly constructed
 `FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
 does not change it.
 For a spread, every leg starts with:
 ```cpp
 leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
 leg.key = invalid_security_key;
 ```
 Thus both the parent and legs initially carry exchange-routing identity, but no
 Spark identity.
 ### 3. Shared-memory allocation and exchange indexing
 `MDP3Listener::add_security()` performs these operations in order:
 1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
   cold entry and copies the security into the cold segment.
 2. Because the key is `0`, `MDProducer` queues the handle through
   `SecurityIDRecorder::ensure_key()`.
 3. `_index.add_security(security.numeric_id, handle)` immediately creates the
   CME-ID-to-handle mapping.
 4. The initial exchange status is written through the handle.
 This ordering means incoming books can already be routed by CME ID while the
 Spark key is pending. Shared-memory consumers cannot discover the security yet.
 Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
 when `key() != 0`, so a trade received in this short window updates the cold
 last-trade state but is not added to the keyed trade log.
 ### 4. Recorder tick: spreads, handles, then database
 Every 200 ms the recorder calls, in this exact order:
 1. `work_spreads()` tries to fill previously unresolved spread leg keys by
   asking `CMESecurityIndex` for each leg exchange ID.
 2. `work_handles()` drains new handles. Outrights are converted to an API
   security and queued for DB lookup immediately. A spread without all leg keys
   is retained in `pending_handle` and its leg pointers are registered in
   `unresolved_legs`.
 3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
 Because spread work precedes new-handle work, a newly seen unresolved spread
 normally waits until at least the next timer tick to resolve legs. It waits
 longer if any outright leg has not yet received its Spark key. A spread's own
 key is not requested until every leg key is non-zero.
 ### 5. Lookup or generation
 `work_db()` first queries `(ns, key)`:
 - Existing row: use its `rowid` as the key and invoke the callback.
 - Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
  rowid inside a transaction, the row object receives that ID, and the callback
  runs after the transaction completes.
 New inserts are throttled to roughly 200 per timer pass (the current `> 200`
 check can admit 201) to avoid delaying heartbeats during startup. The queues are
 TBB concurrent queues, but all SQLite access is serialized onto the recorder's
 dispatcher thread. `check_access()` asserts this single-thread rule after the
 first work pass records the dispatcher thread ID.
 ### 6. Final assignment and visibility
 The queued callback captures a copy of `MDProducerHandle` and calls
 `handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
 then increments `security_count`. From this point:
 - shared-memory scans report the security;
 - `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
 - trades and RFQs can be emitted with a non-zero key;
 - resolved spreads can copy this key into their leg structures.
 ### 7. Sunday preload is a fast path
 For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
 definitions from `securities.sqlite`. It converts them with
 `set_security_key=true`, so both the security key and spread leg keys come from
 the serialized API security. `MDProducer::add_security()` calls `set_key()`
 immediately, and no recorder lookup is required for that preloaded handle. The
 CME exchange ID is still restored into `numeric_id`, allowing the normal CME
 index to be built.
 ## Generalized/newer pipeline in detail
 ### 1. Topology and ownership
 `Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
 `run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
 `MarketDataGateway`, and starts the configured providers. For each `(venue,
 source)` returned by a provider, the gateway creates an `MdSink` containing:
 - a venue-local `GeneralSecurityIndex`;
 - a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
 - a `GeneralMarketDataListener` connecting the source to that producer/index.
 The source decides its callback thread. Some sources call the listener while
 starting; others own network or replay threads. The generalized listener calls
 the producer inline on that source thread. The process-wide `IDGenerator`
 serializes its own calls with a mutex, but the listener's transient
 `cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
 expected to serialize security/event callbacks.
 | Generalized stage | Executing thread | DB call? |
 |---|---|---|
 | Build/parse domain definition | Source/adaptor thread | No |
 | `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
 | `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
 | Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
 | `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
 | Add `GeneralSecurityIndex` entry | Same source callback thread | No |
 There are two important compatibility exceptions inside the provider framework:
 - `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
  legacy flow documented above.
 - `EmumbaDeribitSubscription::start()` also ignores that listener and uses
  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
  producer/index. Its assignment is still synchronous and pre-populated, so it
  resembles the newer ID flow even though it does not pass through
  `GeneralMarketDataListener`.
 ### 2. How a new ID is constructed
 For a normal live generalized adapter, ID creation happens **in the adapter,
 before** `GeneralMarketDataListener::on_securities()`:
 ```text
 logical identity = (domain venue, domain security type, adapter-chosen name)
 Spark ID         = SQLite rowid for that identity in key_ids.sqlite
 ```
 `IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
 the enum prefixes:
 ```text
 VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
 ```
 It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
 row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
 next rowid. That integer is wrapped in `domain::SecurityID` and placed in
 `SecurityCommon::id`.
 The third identity component is adapter-specific, and this matters when trying
 to reproduce a mapping:
 | Source/adaptor family | Typical `name` sent to `IDGenerator` |
 |---|---|
 | Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
 | Taifex | product/instrument ID string |
 | Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
 | Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
 | Emumba Deribit | extracted Emumba security descriptor |
 | Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
 So “security name” alone is not a universal logical key. Reproduction requires
 the exact venue enum, exact security-type enum, and the exact adapter-specific
 name used at insertion time.
 ### 3. What `GeneralMarketDataListener` does with that ID
 `on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
 behavior is controlled by:
 ```cpp
 const bool use_given_sec_key = true;
 ```
 Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
 call the DB. It copies `security_common.id.value()` into its transient
 `cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
 `_add_spot()` then calls `_copy_security_common()`, which sets:
 ```cpp
 to.key = incoming_id;
 to.numeric_id = incoming_id;
 ```
 This collapses the generalized source/correlation ID and Spark shared-memory
 key into the same number. The source's text `exchange_sec_id` remains on the
 domain security but is not copied into `FixedSecurity::numeric_id` by this
 path.
 `_commit_security()` then:
 1. calls `MDProducer::add_security()` with an already non-zero key;
 2. causes `MDProducerHandle::set_key()` to run immediately during that call;
 3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
   number); and
 4. returns with the security already visible in shared memory.
 Because the sink's producer has no recorder, supplying `id == 0` would allocate
 the shared-memory entry but never assign or publish its key. Generalized live
 sources must therefore provide a valid non-zero ID before `on_securities()`.
 The listener contains an alternative branch for `use_given_sec_key == false`.
 That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
 sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
 also effectively unused. Turning the flag off is not a mechanical change: an
 adapter such as Databento allocates using numeric instrument ID while the
 listener branch would use `sec.desc()`, potentially creating a second key for
 the same instrument.
 There is also a spread failure edge case. `_find_sec(sec, true)` records the
 parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
 ID is absent, `_add_spread()` logs and returns without committing the spread. A
 later repeat of the parent definition is then classified as a duplicate and
 ignored, so definition ordering is a hard requirement in the current code.
 ### 4. How later books, trades, and statistics find the security
 Every later domain event carries a `domain::SecurityID`:
 - `Book::security_id()`;
 - `TradeRecord::sec_id`; or
 - `SecurityMarketDataStatistics::security_id`.
 The listener first uses `cc_sec_id_map` to translate that event ID to the value
 stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
 `use_given_sec_key == true`, this mapping is numerically identity-to-identity,
 but it still serves as the admission check that the definition was seen.
 Most high-throughput adapters avoid SQLite on market-data events:
 - Databento allocates on the definition and caches
  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
 - Taifex keeps the resolved ID in its security-details mapper.
 - Other native subscriptions maintain comparable descriptor/ID maps.
 The Python bridge is a notable exception: `to_bts_security_()`,
 `to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
 `IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
 one SQLite `SELECT` for each converted Python definition, trade, or book. It
 returns the existing ID after the first insert, but it is still a DB call and
 takes the global `IDGenerator` mutex on every event.
 ### 5. Persistence timing and latency
 The newer allocator is synchronous:
 - Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
  thread.
 - New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
  (SQLite autocommit), also on the caller's thread.
 - Listener publication: no further DB call under the current
  `use_given_sec_key` setting.
 The ID therefore exists in `key_ids.sqlite` before the adapter sends the
 definition to the listener. There is no intentional 200 ms publication delay,
 but the source callback/startup path directly pays DB and mutex latency. Bulk
 definition discovery performs one lookup and possibly one insert per security;
 `IDGenerator` does not batch those operations.
 WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
 not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
 occasional file-system work. The impact is usually a startup/definition cost
 when adapters cache the result, but it becomes hot-path latency for code such as
 the current Python bridge that resolves every event through the DB.
 `GeneralMarketDataListener` itself does not persist the full contract definition
 to `securities.sqlite`; that remains the separate security-cache/metadata
 persistence path described earlier.
 ## Comparison: MDP3 recorder vs generalized allocator
 | Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
 |---|---|---|
 | Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
 | Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
 | Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
 | DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
 | Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
 | New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
 | Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
 | Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
 | Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
 | Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
 | Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
 | Batching | Yes for new rows | No |
 | Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
 Neither algorithm generates a UUID or a content hash. Both ultimately use an
 opaque, database-assigned SQLite rowid. It is normally increasing and looks like
 a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
 used to *look up* the row is logical; the resulting number is not derivable from
 that tuple.
 ## Can another system reconstruct the same keys?
 Not from `(venue, type, name)` alone. Another system can reconstruct the
 **logical lookup tuple**, but it cannot calculate the numeric Spark key because
 the key depends on the historical insertion order of every row in
 `key_ids.sqlite`.
 For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
 row and receive different IDs if other securities were inserted before it on
 one system. They are sequential opaque IDs, not deterministic hashes and not
 UUIDs.
 Ways to preserve cross-system identity are:
 1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
   identical. This preserves existing numeric keys but requires coordinated
   allocation for new ones.
 2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
   import rows with their original rowids before either system allocates new
   securities.
 3. Treat a stable external identity (venue + type + exchange instrument ID, or
   another canonical instrument identifier) as the cross-system key and carry a
   translation table to each system's local Spark key.
 4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
   be computed independently, but collision handling, descriptor
   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
   with orders/recordings/databases all need explicit design.
 Copying only `securities.sqlite` is insufficient as an allocator strategy. It
 contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
 the mapping used to find or allocate those IDs. Conversely, copying only
 `key_ids.sqlite` preserves the numbers but not all contract metadata.
 ## Other assignment paths in `sparkd`
 The two main pipelines reduce to four ways a producer handle can acquire a key:
 1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
   Sunday preload, relay playback, and other replay paths use this mechanism.
 2. **Recorder-backed live input.** The input key is zero and the producer owns a
   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
   `numeric_id` and otherwise follows the same recorder/index design.
 3. **Service assignment.** `SyntheticSecurityService` calls
   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
   callback receives a copied `api::Security` with its top-level `id` set, then
   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
   rowid allocator.
 4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
   never becomes public. This is not a valid production assignment path, but it
   is the failure mode if a generalized source omits its ID.
 Relay and replay deserve care when reasoning about persistence:
 - The generalized `MDReplayer` forwards the recorded domain security ID and
  emits later books/statistics with the recorded `sec_key`, so the listener's
  current use-given mode preserves the recording's numeric identity.
 - The legacy relay producer converts a recorded `api::Security` with
  `set_security_key=true`, so its shared-memory publication also preserves the
  embedded key.
 - The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
  separate mapping population step, not a deterministic derivation of the key.
 ## Operational implications
 - Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
  definition continuity matter.
 - Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
  another host, even when all instruments are identical.
 - Before moving a feed between adapters, compare the exact `name` each adapter
  supplies to `IDGenerator`. Matching venue/type is insufficient.
 - Cache generated IDs in the source after definition processing. Avoid calling
  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
  this guideline.
 - Keep generated rowids below the 32-bit shared-memory ceiling until the
  `security_key` ABI is widened or explicit bounds checking is added to the
  generalized path.
 - Ensure spread legs are announced before generalized spread parents.
 - Treat `synchronous=0` as a durability decision. A copied DB should be taken
  with SQLite-aware backup/checkpoint handling rather than by casually copying
  active WAL files.
 Useful inspection queries (run against the configured DB path) are:
 ```sql
 SELECT rowid AS spark_key, ns, key
 FROM store
 ORDER BY rowid;
 SELECT rowid AS spark_key, ns, key
 FROM store
 WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
 ```
 ## Source map
 - [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
  and spread-leg resolution.
 - [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
  synchronous generalized lookup/allocation and process-wide mutex.
 - [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
  location, schema, WAL/synchronous pragmas, individual inserts, and batch
  inserts.
 - [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
  shared-memory allocation, recorder handoff, and ordered publication.
 - [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
  widths and hot/cold storage fields.
 - [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
  and
  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
 - [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
  producer/index insertion order.
 - [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
  current dedicated MDP3 dispatcher-thread ownership.
 - [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
  and
  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
  generalized source/sink topology, incoming-ID mapping, fixed-security
  conversion, and publication.
 - [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
  and
  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
  representative adapter-side allocation and caching behavior.
 - [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
  persistence of full security definitions.
 - [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
  translation followed by Spark-keyed market-data consumption and MBO access.
 - [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
  representative downstream use of Spark security keys.
 - [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
  ID reuse.
@@ -19,11 +19,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
  && apt-get install -y dotnet8
 RUN apt-get update && apt-get install -y gdb
 RUN apt-get update && apt-get install -y fish
 RUN apt-get update && apt-get install -y vim
 RUN apt-get update && apt-get install -y less
 RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
@@ -49,6 +49,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{00D21F82-61F4-467F-B125-268018A185E5}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{513A3F0D-46B5-4AC2-B700-8671933F6347}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{D419D996-9870-4FF5-82AD-08370D1C945F}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -275,6 +283,102 @@ Global
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
 		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.ActiveCfg = Release|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.Build.0 = Release|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.Build.0 = Release|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.ActiveCfg = Release|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.Build.0 = Release|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.ActiveCfg = Release|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.Build.0 = Release|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.Build.0 = Release|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.ActiveCfg = Release|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.Build.0 = Release|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.ActiveCfg = Release|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.Build.0 = Release|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.Build.0 = Release|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.ActiveCfg = Release|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.Build.0 = Release|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.ActiveCfg = Release|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.Build.0 = Release|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.Build.0 = Release|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.ActiveCfg = Release|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.Build.0 = Release|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -302,5 +406,9 @@ Global
 		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
 		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{00D21F82-61F4-467F-B125-268018A185E5} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 		{513A3F0D-46B5-4AC2-B700-8671933F6347} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 		{D419D996-9870-4FF5-82AD-08370D1C945F} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 	EndGlobalSection
 EndGlobal
@@ -102,10 +102,10 @@ namespace BCG
         List<ColumnInfo> col_info = new List<ColumnInfo>();
         col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
         foreach (Pair<string, Type> p in group_by_col_info)
-            col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
+            col_info.Add(new ColumnInfo(p.first ?? string.Empty, col_index++, ExceptionAssert.NotNull(p.second)));
         foreach (string tc_prefix in tc_prefixes)
            foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, ExceptionAssert.NotNull(p.second)));
         IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
         if (_schema == null)
            _schema = schema;
@@ -182,7 +182,7 @@ namespace BCG
         if (_transpsed_col_value_comparer_wrapper != null)
            tc_values.Sort(_transpsed_col_value_comparer_wrapper);
         List<string> prefixes = new List<string>();
-         foreach (Pair<int, object[]> p in tc_values)
+         foreach (Pair<int, object?[]> p in tc_values)
            prefixes.Add(tc_prefixes[p.first]);
         _set_schema(prefixes, group_by_col_info, table_name);
         BlueTable result = new BlueTable(_schema);
@@ -1,17 +1,17 @@
 using Spark.Core.Diagnostics;
 using System;
 using System.Collections;
 using System.Diagnostics.CodeAnalysis;
 using System.Text;
 using BCG;
 using BCG.Utilities;
 namespace BCG
 {
   public class ConcatenateColumnsMapper : IStreamMapper
   {
      protected IDictionary myTempColMap = new SortedList ();
-      protected ArrayList myColMap;
+      protected ArrayList? myColMap;
-      protected BlueTableSchema mySchema;
+      protected BlueTableSchema? mySchema;
      public ConcatenateColumnsMapper(params string [] columnDefs)
      {
@@ -55,11 +55,7 @@ namespace BCG
            myColMap = new ArrayList (myTempColMap.Count);
            foreach (DictionaryEntry de in myTempColMap)
            {
-               string[] cols = de.Value as string[];
+               myColMap.Add(Array.ConvertAll((string[])de.Value!, c => inputSchema[c].colNo));
               ArrayList temp = new ArrayList ();
               foreach (string col in cols)
                  temp.Add (inputSchema[col].colNo);
               myColMap.Add (temp.ToArray(typeof(int)) as int[]);
            }
            myTempColMap.Clear ();
         }
@@ -72,6 +68,8 @@ namespace BCG
      public virtual IBlueRow map (IBlueRow row)
      {
         requireSchema();
         requireColMap();
         object[] retVal = new object[mySchema.columnInfo.Length];
         row.CopyTo (retVal, myColMap.Count);
         for (int colNo = 0; colNo < myColMap.Count; colNo ++)
@@ -90,11 +88,21 @@ namespace BCG
      /// <returns></returns>
      protected virtual object myGetConcatValue (IBlueRow row, int colNo)
      {
         requireColMap();
         StringBuilder b = new StringBuilder();
-         foreach (int cn in myColMap[colNo] as int[])
+         foreach (int cn in (int[])myColMap[colNo]!) {
-            b.Append(row[cn].ToString());
+            b.Append(row[cn]?.ToString());
         }
         return b.ToString ();
      }
      [MemberNotNull(nameof(myColMap))]
      protected void requireColMap() =>
         ExceptionAssert.NotNull(myColMap);
      [MemberNotNull(nameof(mySchema))]
      void requireSchema() =>
         ExceptionAssert.NotNull(mySchema);
   }
   public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper 
@@ -107,23 +115,21 @@ namespace BCG
         myDicts = new HashTableND [columnDefs.Length];
         int i = 0;
         foreach (DictionaryEntry de in myTempColMap)
-            myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
+            myDicts[i++] = new HashTableND(((string[])de.Value!).Length);
      }
      protected override object myGetConcatValue (IBlueRow row, int colNo)
      {
-         int[] cols = myColMap[colNo] as int[];
+         requireColMap();
         object[] vals = new object[cols.Length];
         for (int i = 0; i < vals.Length; i++)
            vals[i] = row[cols[i]];
         HashTableND ht = myDicts[colNo];
-         object rv = ht.getValue (vals);
+         var vals = Array.ConvertAll((int[])myColMap[colNo]!, c => row[c]);
         var rv = ht.getValue(vals);
         if (rv != null)
            return rv;
         else
         {
            rv = base.myGetConcatValue (row, colNo);
-            object[] args = new object[vals.Length + 1];
+            var args = new object?[vals.Length + 1];
            vals.CopyTo (args, 0);
            args[args.Length-1] = rv;
            ht.add (args);
@@ -20,7 +20,7 @@ namespace BCG
        {
            IDictionary<string, double> columnSums = new Dictionary<string, double>();
-            IBlueRow resultRow = null;
+            IBlueRow? resultRow = null;
            foreach (IBlueRow row in rows)
            {
                resultRow = row;
@@ -29,7 +29,7 @@ namespace BCG
                {
                    double sum = 0.0f;
                    columnSums.TryGetValue(colName, out sum);
-                    sum += (double) row[colName];
+                    sum += (double)row[colName]!;
                    columnSums[colName] = sum;
                }
            }
@@ -1,8 +1,3 @@
 using System;
 using System.Collections;
 using BCG.Utilities;
 namespace BCG
 {
 	public class DistinctRowEvaluator : GroupByEvaluator
@@ -67,7 +67,7 @@ namespace BCG {
      }
      #endregion
-      public event FilterChangeDelegate onFilterChange;
+      public event FilterChangeDelegate? onFilterChange;
      #region IStreamFilter
      public bool filter(IBlueRow row) {
@@ -43,7 +43,7 @@ namespace bts.SilverPine {
         operations = operations_;
      }
-      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+      public static IBlueRow? group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
         if (rows.Count() == 0) {
            return null;
         }
@@ -73,7 +73,7 @@ namespace bts.SilverPine {
      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
-      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
         if (rolling_rows.Count() == 0) {
            return null;
         }
@@ -123,4 +123,4 @@ namespace bts.SilverPine {
      }
   }
-}
+}
@@ -1,4 +1,4 @@
-using System;
+using Bts.DataStreamCore;
 namespace BCG
 {
@@ -9,18 +9,18 @@ namespace BCG
 	public abstract class GroupByEvaluator : IGroupByEvaluator
 	{
      protected string[] myGroupByCols;
-      protected string myOutputColumn;
+      protected string? myOutputColumn;
-      protected GroupByOutputTable myOutputTable;
+      protected GroupByOutputTable? myOutputTable;
-      protected string myColMapExpression;
+      protected string? myColMapExpression;
      #region constructors
-		public GroupByEvaluator (string outputCol, string[] groupByCols)
+		public GroupByEvaluator (string? outputCol, string[] groupByCols)
 		{
         myOutputColumn = outputCol;
         myGroupByCols = groupByCols;
 		}
-      public GroupByEvaluator (string colMapExpression, string outputCol, string[] groupByCols)
+      public GroupByEvaluator (string colMapExpression, string? outputCol, string[] groupByCols)
         : this (outputCol, groupByCols)  {myColMapExpression = colMapExpression;}
      #endregion
@@ -52,21 +52,20 @@ namespace BCG
      /// <param name="r"></param>
      /// <param name="id"></param>
      /// <returns></returns>
-      protected IBlueRow myProject (IBlueRow r, int id)
+      protected IBlueRow? myProject(IBlueRow? r, int id) =>
-      {
+         r is null
-         return r == null ? null :
+            ? null
-            myColMapExpression == null || myColMapExpression.Length < 1 ?
+            : myColMapExpression is null || myColMapExpression.Length < 1
-               BlueDataUtility.getNewReplacedRow (r, id, "id") :
+               ? BlueDataUtility.getNewReplacedRow(r, id, "id")
-               BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression),
+               : BlueDataUtility.getNewReplacedRow(myProjectRow(r, myColMapExpression), id, "id");
-            id, "id");
+
      }
      /// <summary>
      /// Projects the id, conserving the id
      /// </summary>
      /// <param name="r"></param>
      /// <returns></returns>
-      protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
+      protected IBlueRow? myProject(IBlueRow? r) =>
         myProject(r, r is null ? int.MinValue : r.Id());
      #endregion
      IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
@@ -1,5 +1,6 @@
 using Bts.DataStreamCore;
 using System;
 using BCG.Utilities;
 using System.Collections;
 namespace BCG
@@ -22,10 +23,12 @@ namespace BCG
      /// <param name="groupRepString"></param>
      public void addRow (IBlueRow row, string groupRepString)
      {
-         if (myDict[groupRepString] != null)
+         var grp = myDict[groupRepString];
-            myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
+         if (grp is not null) {
            myTable.deleteRow((IBlueRow)grp);
         }
         myTable.addRow (row);
-         myDict[groupRepString] = myTable[(int)row["id"]];
+         myDict[groupRepString] = myTable[row.Id()];
      }
      /// <summary>
@@ -35,7 +38,7 @@ namespace BCG
      public void removeRow (string groupRepString)
      {
         // BlueEventLog.logAssert (myDict[groupRepString] != null);
-         myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
+         myTable.deleteRow ((IBlueRow)myDict[groupRepString]!);
         myDict.Remove (groupRepString);
      }
@@ -48,11 +51,8 @@ namespace BCG
         myDict.Clear ();
      }
-      public IBlueRow getRow (string groupRepString)  
+      public IBlueRow? getRow(string groupRepString) =>
-      {
+         myDict[groupRepString] as IBlueRow;
         return myDict[groupRepString] == null ? null : 
            myDict[groupRepString] as IBlueRow;
      }
      #region IBlueRowCollection members
      public IBlueRow this[int id]  {get{return myTable[id];}}
@@ -1,7 +1,9 @@
 using Spark.Core.Diagnostics;
 using System;
 using System.Collections;
 using BCG.Utilities;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 namespace BCG {
@@ -26,11 +28,12 @@ namespace BCG {
      /// Serves as  a quick way to access all rows belonging to the same group
      /// </summary>
      private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable;
+      RowDictionaryTable? myBookKeepingTable;
      private IGroupByEvaluator myGroupByEvaluator;
-      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
+      // input schema, as output schema can differ (IGroupByEvaluator's
      IBlueTableSchema? mySchema;
      //can map a schema to another
-      private IBlueTableSchema myOutputSchema;
+      IBlueTableSchema? myOutputSchema;
      private int myIdGenerator = 0;
      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
@@ -46,14 +49,14 @@ namespace BCG {
      }
      protected override void myHandleFirstCall() {
         requireLastArrivedPacket();
         mySchema = myLastArrivedPacket.schema;
         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
            myGroupByEvaluator.groupByColumns);
         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+            var newRow = myEvaluate(new DataPacket(rowGroup, rowGroup, new BlueTable(mySchema)));
               rowGroup, new BlueTable(mySchema)));
            if (newRow != null)
               temp.addRow(newRow, myGetGroupString(rowGroup));
         }
@@ -79,15 +82,14 @@ namespace BCG {
         ArrayList handledAddGroups = new ArrayList();
         string groupRepString;
         requireBookKeeping();
         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
            // BlueEventLog.logAssert (delGroup.Count > 0);
            groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) ?? new BlueTable(mySchema);
               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
            handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) ?? new BlueTable(mySchema);
-               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
            myFireSwitch |= myMergeRow(newRow, groupRepString);
         }
@@ -99,16 +101,16 @@ namespace BCG {
               //the loop above
               groupRepString = myGetGroupString(addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
+               var aggGroup = ExceptionAssert.NotNull(myBookKeepingTable.getGroup(groupRepString));
               // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+               var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, new BlueTable(aggGroup.schema)));
                  new BlueTable(aggGroup.schema)));
               myFireSwitch |= myMergeRow(newRow, groupRepString);
            }
         }
      }
-      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+      private bool myMergeRow(IBlueRow? newRow, string groupRepString) {
         requireLastFiredPacket();
         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
         //BlueEventLog.logAssert (newRow != null || existingRow != null);
         if (existingRow == null && newRow == null)
@@ -116,6 +118,7 @@ namespace BCG {
         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
            return false;
         else if (existingRow == null) {
            ArgumentNullException.ThrowIfNull(newRow);
            //row added for the first time
            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
@@ -138,6 +141,9 @@ namespace BCG {
      /// <param name="delGroups"></param>
      /// <param name="addGroups"></param>
      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
         ExceptionAssert.NotNull(myBookKeepingTable);
         requireLastFiredPacket();
         requireLastArrivedPacket();
         delGroups =
            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
         addGroups =
@@ -151,16 +157,19 @@ namespace BCG {
      }
      protected override void myOnTableClear() {
-         if (myBookKeepingTable.Count == 0)
+         if (ExceptionAssert.NotNull(myBookKeepingTable).Count == 0) {
-            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
+            // ie there arent any rows, so no need to optimize the 'cleansing' process
            return;  
         }
         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         ((GroupByOutputTable)myLastFiredPacket.newRows).clear();
         myBookKeepingTable.clear();
         myGroupByEvaluator.clear();
      }
-      private IBlueRow myEvaluate(DataPacket dp) {
+      private IBlueRow? myEvaluate(DataPacket dp) {
         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
         return evalRow == null ? null :
            myIdGuard ?
@@ -192,7 +201,7 @@ namespace BCG {
         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
      }
-      public static IBlueRow getRepRow(IDataPacket dp) {
+      public static IBlueRow? getRepRow(IDataPacket dp) {
         if (dp.newRows.Count > 0)
            return BlueDataUtility.extractRow(dp.newRows, 0);
         else if (dp.addedRows.Count > 0)
@@ -211,13 +220,13 @@ namespace BCG {
      /// <param name="newValue"></param>
      /// <param name="isAdd"></param>
      /// <returns></returns>
-      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+      public static object? incrementalAdd(object lastValue, object newValue, bool isAdd) {
         if (lastValue == null || lastValue.Equals(DBNull.Value))
            return newValue;
         // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
-         Object retVal = null;
+         object? retVal = null;
         if (newValue.GetType().Equals(typeof(int)))
            retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
         else if (newValue.GetType().Equals(typeof(Double)))
@@ -229,6 +238,9 @@ namespace BCG {
         return retVal;
      }
      [MemberNotNull(nameof(myBookKeepingTable))]
      void requireBookKeeping() =>
         ExceptionAssert.NotNull(myBookKeepingTable);
   }
 }
@@ -1,8 +1,3 @@
 using System;
 using System.Collections;
 using BCG.Utilities;
 namespace BCG {
    /// <summary>
    /// Computes the max for a given column
@@ -17,10 +12,10 @@ namespace BCG {
        public override IBlueRow? evaluate(IDataPacket dp) {
            myOutputTable ??= new GroupByOutputTable(dp.Schema);
            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            var currMaxRow = myOutputTable.getRow(hashString);
            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
            if (maxRow != null) {
-                bts.Log.assert(currMaxRow != null);
+                bts.Log.notnull(currMaxRow);
                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
@@ -1,10 +1,9 @@
 using System;
 using System.Collections;
 using BCG.Utilities;
 using System.IO;
 namespace BCG
 {
   using Spark.Core.Diagnostics;
   using System.Diagnostics.CodeAnalysis;
   /// <summary>
   /// Some basic event handling functionality factorred out of any Stream whose behaviour
   /// is driven by some interface passed on by the user
@@ -12,8 +11,8 @@ namespace BCG
   public abstract class MethodDrivenStream : ViewStream
   {
      protected ViewStream myStream;
-      protected DataPacket myLastArrivedPacket;
+      protected DataPacket? myLastArrivedPacket;
-      protected DataPacket myLastFiredPacket;
+      protected DataPacket? myLastFiredPacket;
      protected bool myFireSwitch = true;
      public MethodDrivenStream(ViewStream vs)  {myStream = vs;}
@@ -59,6 +58,7 @@ namespace BCG
      /// To handle the first call, the LastFiredPacket is supposed to be initialized and filled
      /// with data.
      /// </summary>
      [MemberNotNull(nameof(myLastFiredPacket))]
      protected abstract void myHandleFirstCall ();
      /// <summary>
@@ -89,5 +89,13 @@ namespace BCG
      {
         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
      }
      [MemberNotNull(nameof(myLastArrivedPacket))]
      protected void requireLastArrivedPacket() =>
         ExceptionAssert.NotNull(myLastArrivedPacket);
      [MemberNotNull(nameof(myLastFiredPacket))]
      protected void requireLastFiredPacket() =>
         ExceptionAssert.NotNull(myLastFiredPacket);
   }
 }
@@ -9,7 +9,7 @@ namespace BCG {
        public override IBlueRow? evaluate(IDataPacket dp) {
            myOutputTable ??= new GroupByOutputTable(dp.Schema);
            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            var currMinRow = myOutputTable.getRow(stringRep);
            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
            if (minRow != null) {
                // BlueEventLog.logAssert (currMinRow != null);
@@ -45,12 +45,13 @@ namespace BCG {
        }
        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
-            ViewStream rs = null;
+            ViewStream? rs = null;
            groupby_columns = groupby_columns ?? new string[] { };
            if (groupby_columns.Length == 0) {
                // rs = new AddColumnsStream(stream, "fake_col=0");
                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
                // rs = new ProjectStream(rs, "!fake_col");
                throw new ArgumentException("No groupby columns specified", nameof(groupby_columns));
            } else {
                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
            }
@@ -1,5 +1,7 @@
 using mbts;
 using Spark.Core.Diagnostics;
 using System;
 using System.Collections.Generic;
@@ -35,7 +37,7 @@ namespace BCG {
         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
         foreach (Triplet<ViewStream, string, string> r in join_specs) {
            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
+            string[] visible_cols = ExceptionAssert.NotNull(r.third).Split(' ');
            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
            foreach (string s in visible_cols) {
               string[] toks = s.Split(':');
@@ -47,7 +49,7 @@ namespace BCG {
                  mapped_col_names.Add(toks[1]);
               }
            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add(new(ExceptionAssert.NotNull(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
         }
         return rv.ToArray();
      }
@@ -290,6 +290,7 @@ namespace BCG
               }
            }
         }
         ExceptionAssert.NotNull(myState);
      }
      private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
@@ -1,3 +1,5 @@
 using Spark.Core.Diagnostics;
 using System;
 namespace BCG
@@ -18,16 +20,21 @@ namespace BCG
      public override void onTableClear (bool isLeft)
      {
         requireSchema();
         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
         myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
         myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft)
+         if (!isLeft) {
            ExceptionAssert.NotNull(myLeftDataPacket);
            foreach (IBlueRow row in myLeftDataPacket.newRows)
               ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
         }
      }
      protected override void myInitFiredPacket ()
      {
         requireSchema();
         myLastFiredPacket = new DataPacket ();
         myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
         myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -15,7 +15,7 @@ namespace BCG
         : this (r, null, null)
      {}
-      public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
+      public SimpleNode(IBlueRow? r, SimpleNode? nx, SimpleNode? pv) {
         row = r;
         next = nx;
         prev = pv;
@@ -1,5 +1,7 @@
 using bts;
 using Spark.Core.Diagnostics;
 using System;
 using System.Collections.Generic;
@@ -60,7 +62,7 @@ namespace BCG {
            if (columnAndType.Length == 1)
               ci[n] = new ColumnInfo(c, n, typeof(double));
            else
-               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+               ci[n] = new(columnAndType[0], n, ExceptionAssert.NotNull(Type.GetType("System." + columnAndType[1], true)));
            n++;
         }
@@ -44,7 +44,7 @@ namespace bts {
         BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
         :
         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
-         BlueTable out_table = null;
+         BlueTable? out_table = null;
         foreach (var kv in groups) {
            List<IBlueRow> group_rows = kv.Value;
            group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
@@ -73,4 +73,4 @@ namespace bts {
   }
-}
+}
@@ -23,6 +23,6 @@ namespace BCG
      /// fires an event when the mapper 
      /// changes itself
      /// </summary>
-      event MapperChangeDelegate onMapperChange;
+      event MapperChangeDelegate? onMapperChange;
   }
 }
@@ -75,6 +75,7 @@ namespace BCG {
      protected override void myHandleFirstCall() {
         ArgumentNullException.ThrowIfNull(myMapper);
         requireLastArrivedPacket();
         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
         myLastFiredPacket = new DataPacket();
         myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -85,10 +86,12 @@ namespace BCG {
      }
      protected override void myHandleIncrementalAdd() {
         requireLastArrivedPacket();
         if (myLastArrivedPacket.addedRows.Count == 0) {
            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
            return;
         }
         requireLastFiredPacket();
         ArgumentNullException.ThrowIfNull(myMapper);
         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
@@ -101,10 +104,12 @@ namespace BCG {
      }
      protected override void myHandleIncrementalDelete() {
         requireLastArrivedPacket();
         if (myLastArrivedPacket.deletedRows.Count == 0) {
            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
            return;
         }
         requireLastFiredPacket();
         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
@@ -127,6 +132,7 @@ namespace BCG {
      }
      protected override void myOnTableClear() {
         requireLastFiredPacket();
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
@@ -47,14 +47,14 @@ namespace BCG {
            _map_rows = map_rows;
         }
         event MapperChangeDelegate? ISuperStreamMapper.onMapperChange { add { } remove { } }
         public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
         public IBlueRow[] map(IBlueRow r) {
            IEnumerable<IBlueRow> rows = _map_rows(r);
            return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
         }
         public event MapperChangeDelegate? onMapperChange;
      }
      //maps the id of the underlying stream to the
@@ -608,8 +608,8 @@ namespace BCG {
            // 5. deserialize the state into a stream op
            str_io_msg_p = deser.deserialize(content_p);
            // 6. check to make sure the state's rows are good
-            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
+            passes &= str_io_msg_p.op == StreamIOMessage.StreamOp.CREATE
-                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
+               && BlueDataUtility.compute_equality(cr_rows, ExceptionAssert.NotNull(str_io_msg_p.create_message).rows);
            return new TestResult {
                test_result = passes,
                test_name = "run_pubsub_server_stream_tag_handler_test"
@@ -692,7 +692,7 @@ namespace BCG {
            var content = st_data.get_current_content();
            var msg_p = deser.deserialize(content);
            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
-            return msg_p.create_message.rows;
+            return ExceptionAssert.NotNull(msg_p.create_message).rows;
        }
@@ -1,5 +1,7 @@
 using Bts.ColumnSortExpressions;
 using Spark.Core.Diagnostics;
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
         var otherVal = otherRow[col.Name];
         int rv = myIsNull(this[col.Name]) ?
            (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
+            (myIsNull(otherVal) ? 1 : ExceptionAssert.Argument.Is<IComparable>(this[col.Name]).CompareTo(otherVal));
         rv *= col.IsAscending ? 1 : -1;
         return rv;
      }
@@ -1,4 +1,6 @@
-using System;
+using Spark.Core.Diagnostics;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -60,13 +62,13 @@ namespace BCG
            List<string> columnInfoStrParts = new List<string>(columnInfoStr.Split(new char[] { ':' }, StringSplitOptions.None));
            if (columnInfoStrParts.Count != 2)
            {
-               throw (new Exception(String.Format("Unable to decode schema string '{0}'", schemaStr)));
+               throw new Exception($"Unable to decode schema string '{schemaStr}'");
            }
-            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, Type.GetType(columnInfoStrParts[1])));
+            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, ExceptionAssert.NotNull(Type.GetType(columnInfoStrParts[1]))));
         }
-         return (new BlueTableSchema(columnInfos.ToArray(), ""));
+         return new BlueTableSchema(columnInfos.ToArray(), string.Empty);
      }
      private IBlueRow myGetRow(IBlueTableSchema schema, string rowStr)
@@ -75,7 +77,7 @@ namespace BCG
         if (schema.columnInfo.Length != valueStrs.Count)
         {
-            throw (new Exception(String.Format("Unable to decode row string '{0}'", rowStr)));
+            throw new Exception($"Unable to decode row string '{rowStr}'");
         }
         List<object> values = new List<object>();
@@ -84,7 +86,7 @@ namespace BCG
            values.Add(Convert.ChangeType(valueStrs[i], schema.columnInfo[i].colType));
         }
-         return (new BlueRow(schema, values.ToArray()));
+         return new BlueRow(schema, values.ToArray());
      }
      #endregion
@@ -1,6 +1,7 @@
 using BCG.Utilities.General;
 using System;
 using System.Collections.Generic;
 using System.Threading;
 namespace BCG
@@ -97,7 +98,7 @@ namespace BCG
         // BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread, 
         //    "onTargetDoWork should only be called from target thread");
         var updates = _queued_updates.dequeueAll();
-         foreach (var u in updates) {
+         foreach (var u in updates.ExceptNull()) {
            if (u.subscription == _current_target_subscription)
               myFireEvent(u.dp);
         }
@@ -1,7 +1,9 @@
 using System;
 using BCG.Utilities.General;
 using Spark.Core.Diagnostics;
 using System;
 namespace BCG {
   public abstract class TimeUnitStream : ViewStream, IWorker {
      protected TimeSpan myOffset;
@@ -46,8 +48,8 @@ namespace BCG {
            myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
-         if (myGetLatestRow() == null ||
+         var latestRow = myGetLatestRow();
-            !myGetLatestRow()["time"].Equals(dt)) {
+         if (latestRow is null || !ExceptionAssert.Argument.Is<DateTime>(latestRow["time"]).Equals(dt)) {
            IBlueRow ar = new BlueRow(myCollection.schema,
               myId++, dt);
            IBlueRow dr = myCollection.addRowWithReturn(ar);
@@ -71,11 +73,10 @@ namespace BCG {
      #region Private Helper methods
-      private IBlueRow myGetLatestRow() {
+      IBlueRow? myGetLatestRow() =>
-         return myCollection.Count == 0 ?
+         myCollection.Count == 0
-            null :
+            ? null
-            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
+            : BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
      }
      private IBlueRowCollection myConvertToCollection(IBlueRow r) {
         BlueTable retVal = new BlueTable(myCollection.schema);
@@ -1,43 +1,42 @@
-using System;
+namespace BCG;
-namespace BCG {
+public class TopStream : MethodDrivenStream {
-    public class TopStream : MethodDrivenStream {
+   int myTopCount;
        private int myTopCount;
-        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
+   public TopStream(ViewStream stream, int topCount)
      : base(stream) =>
      myTopCount = topCount;
-        public TopStream(ViewStream stream,
+   public TopStream(ViewStream stream, int topCount, string sortExpression)
-           int topCount,
+      : this(sortExpression is null ? stream : new SortStream(stream, sortExpression), topCount) { }
           string sortExpression)
           :
           this(sortExpression == null ?
           stream :
           new SortStream(stream, sortExpression), topCount) { }
-        protected override void myOnTableClear() {
+   protected override void myOnTableClear() {
-            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+      requireLastFiredPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+      myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+      myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-        }
+      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
   }
-        protected override void myHandleFirstCall() {
+   protected override void myHandleFirstCall() {
-            myLastFiredPacket = new DataPacket();
+      requireLastArrivedPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+      myLastFiredPacket = new DataPacket();
-            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-        }
+      myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
   }
-        protected override void myHandleIncrementalAdd() { }
+   protected override void myHandleIncrementalAdd() { }
-        protected override void myHandleIncrementalDelete() {
+   protected override void myHandleIncrementalDelete() {
-            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+      requireLastArrivedPacket();
-            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+      requireLastFiredPacket();
-            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+      IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-               oldRows,
+      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-               myLastFiredPacket.schema.columnNames,
+      BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-               out myLastFiredPacket.deletedRows,
+         oldRows,
-               out myLastFiredPacket.addedRows);
+         myLastFiredPacket.schema.columnNames,
-            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+         out myLastFiredPacket.deletedRows,
-        }
+         out myLastFiredPacket.addedRows);
-    }
+      myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
   }
 }
@@ -1,12 +1,10 @@
 using System;
 using System.Collections;
 namespace BCG {
   public class AlphabetizeColumnsStream : ViewStream {
      private ViewStream mySrc;
-      private IBlueTableSchema mySchema = null;
+      IBlueTableSchema? mySchema = null;
      public AlphabetizeColumnsStream(ViewStream src) {
         mySrc = src;
@@ -1,12 +1,12 @@
-using System;
+using System.Linq;
 namespace BCG
 {
 	public class AppendStreamModified : ViewStream
 	{
      private AppendCollection myColl = new AppendCollection ();
-      private ViewStream myLeftStream, myRightstream;
+      ViewStream? myLeftStream, myRightstream;
-	
+
      public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
 		{
         myLeftStream = leftStream;
@@ -15,11 +15,24 @@ namespace BCG
      public AppendStreamModified (ViewStream[] vss)
      {
         switch (vss.Length) {
            case 0:
               break;
            case 1:
               mySetRealSource (vss[0]);
               break;
            default:
               mySetRealSource(
                  vss[1..].Aggregate(
                     new AppendStreamModified(new[] { vss[0] }),
                     (asm, vs) => new AppendStreamModified(asm, vs)));
               break;
         }
         if (vss.Length == 1)
            mySetRealSource (vss[0]);
         else
         {
-            AppendStreamModified rv = null;
+            AppendStreamModified? rv = null;
            foreach (ViewStream s in vss)
               rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s); 
            mySetRealSource (rv);
@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
 using global::Spark.Core.Disposables;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 using Spark.Core.Disposables;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -8,7 +8,6 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;
@@ -43,6 +43,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "..\cs\Bts.Collections\Bts.Collections.csproj", "{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "..\cs\Bts.Core\Bts.Core.csproj", "{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "..\cs\Bts.Disposables\Bts.Disposables.csproj", "{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "..\cs\Bts.Numerics\Bts.Numerics.csproj", "{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -293,6 +301,54 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.Build.0 = Debug|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.Build.0 = Debug|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.Build.0 = Release|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.ActiveCfg = Release|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.Build.0 = Release|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.ActiveCfg = Release|Any CPU
 		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.Build.0 = Release|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.Build.0 = Debug|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.Build.0 = Debug|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.ActiveCfg = Release|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.Build.0 = Release|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.ActiveCfg = Release|Any CPU
 		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.Build.0 = Release|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.Build.0 = Debug|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.Build.0 = Debug|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.ActiveCfg = Release|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.Build.0 = Release|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.ActiveCfg = Release|Any CPU
 		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.Build.0 = Release|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.Build.0 = Debug|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.Build.0 = Debug|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.ActiveCfg = Release|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.Build.0 = Release|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.ActiveCfg = Release|Any CPU
 		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -5,7 +5,7 @@ using BCG.Utilities.General;
 using Bts.DataStreamCore;
-using global::Spark.Core.Collections;
+using Spark.Core.Collections;
 public class TestDriverStream : ViewStream, IWorker {
   readonly IBlueTableSchema schema;
@@ -22,7 +22,6 @@ namespace BCG {
   /// with PubSub, as some clients are still using ManagedSubcribers)
   /// </summary>
   public class ManagedSubscriber : IManagedSubscriber, IWorker, IDisposable {
      private static ManagedSubscriber? _self;
      private ISubscriberCore _subcore;
      // private IDictionary myTypedDispatchTable = new Hashtable(),
      //    myRawDispatchTable = new Hashtable();
@@ -34,7 +33,6 @@ namespace BCG {
      ManagedSubscriber(ISubscriberCore subsCore) { _subcore = subsCore; }
      public void doWork() {
         var msg_map = new Dictionary<string, List<bts.PubSubServerMessage>>();
         var updates = _subcore.get_updates(TimeSpan.Zero);
         if (updates.Count() == 0)
            return;
@@ -45,11 +45,7 @@ namespace BCG
         get { return myConnInfo; }
      }
-      override public string ToString()
+      public override string ToString() =>
-      {
+         $"type: {myMsgType} conn: {myConnInfo} msg: {myMsg?.Length ?? 0}";
         Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
         return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
      }
   }
 }
@@ -46,7 +46,7 @@ namespace bts {
            System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
            _accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { });
            if (ex is OutOfMemoryException)
-               throw ex;
+               throw;
            return;
         }
         new_sock.LingerState = new LingerOption(false, 0);
@@ -1,9 +1,8 @@
 using bts;
 using System;
 using System.IO;
-using System.Collections;
+using System.Text;
 using bts;
 using System.Threading;
 using System.Reflection;
 namespace BCG {
   /// <summary>
@@ -69,7 +68,8 @@ namespace BCG {
      public static byte[] getBytesFromStream(params Stream[] streams) {
         long totalSize = 0;
-         BinaryWriter bw = new BinaryWriter(new MemoryStream());
+         using var ms = new MemoryStream();
         using var bw = new BinaryWriter(ms, Encoding.UTF8, true);
         foreach (Stream s in streams)
            totalSize += s.Length - s.Position;
         bw.Write(Convert.ToInt32(totalSize));
@@ -78,7 +78,8 @@ namespace BCG {
            int lengthToReadForThisStream = Convert.ToInt32(s.Length - s.Position);
            bw.Write(br.ReadBytes(lengthToReadForThisStream));
         }
-         return (bw.BaseStream as MemoryStream).ToArray();
+         bw.Flush();
         return ms.ToArray();
      }
      public static byte[] read_bytes(Stream str) {
@@ -92,7 +92,7 @@ namespace BCG {
      public void publish(PubSubClientMessage msg) {
         Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
-         Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
+         Log.assert(!string.IsNullOrEmpty(msg.pub_msg?.tag_type));
         bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
         _main_queue.enqueue(msg);
      }
@@ -143,7 +143,7 @@ namespace BCG {
      public void doWork() {
         if (_connection_cb != null) {
            var conns = _conn_callback_queue.dequeueAll();
-            foreach (var ci in conns) {
+            foreach (var ci in conns.ExceptNull()) {
               _connection_cb(ci.client_url, ci.status);
            }
         }
@@ -175,7 +175,7 @@ namespace BCG {
            case RemoteMessageType.Data:
               // if callback is registered fire it right away, else enqueue it
               if (_data_cb != null)
-                  _data_cb(sender_rci.ToString(), message.message);
+                  _data_cb(sender_rci.ToString(), message.message ?? Array.Empty<byte>());
               else
                  _received_data_queue.enqueue(new Pair<string, byte[]>(sender_rci.ToString(), message.message));
               break;
@@ -258,7 +258,7 @@ namespace BCG {
      // (which gets called in another thread)
      private void _handle_new_connections() {
         var new_conns = _new_connections.dequeueAll();
-         foreach (ITCPConnection conn in new_conns) {
+         foreach (var conn in new_conns.ExceptNull()) {
            var peer_rci = conn.get_peer_address();
            bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections conn: {conn}");
            ///to stop it from being garbabge collected immediatly
@@ -16,8 +16,8 @@ public interface IPubSubState {
   ICollection<string> all_tags { get; }
-   byte[] get_content(string tag);
+   byte[]? get_content(string tag);
-   string getTagType(string tag);
+   string? getTagType(string tag);
   void get_all_connections(
      string tag,
      out Dictionary<string, string> subscribers,
@@ -188,11 +188,11 @@ namespace BCG {
         _state.handle_sub(sub_msg);   // state keeps the list of all subscribers
         ///to all these new subscriber s, send the existing data
         foreach (string tag in sub_msg.tags.empty_if_null()) {
-            byte[] data = _state.get_content(tag);
+            byte[]? data = _state.get_content(tag);
            //ie data has been published once
            if (data != null) {
               bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
-               string tag_type_ = _state.getTagType(tag);
+               string? tag_type_ = _state.getTagType(tag);
               var msg_to_send = new PubSubServerMessage() {
                  tag = tag,
                  tag_type = tag_type_,
@@ -148,7 +148,7 @@ namespace BCG {
            return new string[] { };
      }
-      public byte[] get_content(string tag) {
+      public byte[]? get_content(string tag) {
         if (!_tag_to_data_map.ContainsKey(tag))
            return null;
         var rv = _tag_to_data_map[tag].get_current_content();
@@ -157,7 +157,7 @@ namespace BCG {
         return rv;
      }
-      public string getTagType(string tag) {
+      public string? getTagType(string tag) {
         if (!_tag_to_data_map.ContainsKey(tag))
            return null;
         else
@@ -20,7 +20,7 @@ namespace bts {
   public class SingleTagData {
      private readonly IPubSubServerTagHandler? _tag_handler;
-      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
+      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler? tag_handler) {
         bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
         this.tag = tag;
         this.tag_type = tag_type;
@@ -46,7 +46,6 @@ namespace BCG {
      }
      public void onConnection(ITCPConnection connection) {
         string ip; int port;
         var peer_rci = connection.get_peer_address();
         bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop");
         if (_listener != null) {
@@ -5,9 +5,10 @@ using BCG;
 using bts.api;
 using bts.md;
 using Bts.DataStream;
 using Bts.DataStreamCore;
-using global::Spark.Api.MarketData;
+using Spark.Api.MarketData;
 public class BookSchemaBuilder {
   public int NumLevels { get; set; } = 5;
@@ -2,7 +2,7 @@ namespace Bts.SharedStreams;
 using bts.api;
-using global::Spark.Api.Orders;
+using Spark.Api.Orders;
 public static class WellKnownColumns {
   public static (string Name, Type Type) AskNotional => (WellKnownColumnNames.AskNotional, typeof(double));
@@ -3,6 +3,7 @@ using bts.config;
 using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 namespace BCG {
@@ -21,7 +22,9 @@ namespace BCG {
    }
    public class RenkoCalculator {
        public readonly double brick_size;
-        public RenkoBrick current_brick = null;
+        public RenkoBrick? current_brick;
        [MemberNotNull(nameof(current_brick))]
        private bool _add_price(double price) {
            if (current_brick == null) {
                current_brick = new RenkoBrick {
@@ -309,4 +312,4 @@ namespace BCG {
            return wma_values;
        }
    }
-}
+}
@@ -10,7 +10,7 @@
   [DataStreamUserRespresentation("nl")]
   public class BlueOrbNLStream : ViewStream {
      private readonly string _nl_prompt;
-      private ViewStream _nl_to_dsl_stream, _dsl_stream;
+      private ViewStream? _nl_to_dsl_stream, _dsl_stream;
      private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
      public BlueOrbNLStream(string nl_prompt) {
         _nl_prompt = nl_prompt;
@@ -85,7 +85,8 @@ namespace bts.BlueOrb {
                .GetProperty("choices")[0]
                .GetProperty("message")
                .GetProperty("content")
-                .GetString();
+                .GetString()
                ?? string.Empty;
            //return gpt_response;
            DateTime t1 = DateTime.Now;
@@ -4,6 +4,8 @@ using BCG.Utilities.General;
 using Bts.PubSub;
 using Bts.SharedStreams.Tables;
 using Spark.Core.Diagnostics;
 namespace bts {
   // TODO: Use dependency injection to instantiate this stream, and others like it that depend on
   // ISubscriberCore(Factory). Until then, hide it from DSQA.
@@ -40,7 +42,7 @@ namespace bts {
         foreach (var u in updates) {
            if (u.tag != "System_AllStatistics")
               continue;
-            var tag_infos = TagStatsInfo.deserialize(u.data);
+            var tag_infos = TagStatsInfo.deserialize(ExceptionAssert.NotNull(u.data));
            foreach (var ti in tag_infos) {
               _tag_map[ti.tag] = ti;
            }
@@ -187,24 +187,32 @@
      Name="Test"
      DependsOnTargets="Build"></Target>
-  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
+  <ItemGroup Condition="Exists('@(NativeWindowsAsset)')">
-    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
+    <Content Include="@(NativeWindowsAsset)">
-    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-  </Target>
+      <TargetPath>%(FileName)%(Extension)</TargetPath>
      <Pack>false</Pack>
    </Content>
    <Content Include="@(NativeWindowsAsset)">
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
      <TargetPath>runtimes/win-x64/native/%(FileName)%(Extension)</TargetPath>
      <Pack>true</Pack>
      <PackagePath>runtimes/win-x64/native</PackagePath>
    </Content>
  </ItemGroup>
-  <ItemGroup>
+  <ItemGroup Condition="Exists('@(NativeLinuxAsset)')">
-    <Content Include="@(NativeWindowsAsset)"
+    <Content Include="@(NativeLinuxAsset)">
-             Condition="Exists('@(NativeWindowsAsset)')"
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
+      <TargetPath>%(FileName)%(Extension)</TargetPath>
-             CopyToOutputDirectory="PreserveNewest"
+      <Pack>false</Pack>
-             Pack="true"
+    </Content>
-             PackagePath="runtimes/win-x64/native/" />
+    <Content Include="@(NativeLinuxAsset)">
-    <Content Include="@(NativeLinuxAsset)"
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-             Condition="Exists('@(NativeLinuxAsset)')"
+      <TargetPath>runtimes/linux-x64/native/%(FileName)%(Extension)</TargetPath>
-             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
+      <Pack>true</Pack>
-             CopyToOutputDirectory="PreserveNewest"
+      <PackagePath>runtimes/linux-x64/native</PackagePath>
-             Pack="true"
+    </Content>
             PackagePath="runtimes/linux-x64/native/" />
  </ItemGroup>
 </Project>
@@ -2,14 +2,14 @@ ARG IMAGE="sparkd"
 ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
 ARG SOURCE_DIR="install/sparkd-release-gcc"
 ARG PYTHON_REQUIREMENTS="requirements.txt"
-ARG SPARKD_ENTRYPOINT="src/app_sparkd/sparkd-entrypoint.sh"
+ARG ENTRYPOINT_SCRIPT="src/app_sparkd/sparkd-entrypoint.sh"
 # sparkd_base
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_base
 ARG CONFIG_DIR
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS
-ARG SPARKD_ENTRYPOINT
+ARG ENTRYPOINT_SCRIPT
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -23,7 +23,7 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
 COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/.local/share/bts/sparkd/conf"
-COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
+COPY --chown=bts:bts "${ENTRYPOINT_SCRIPT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"
 # sparkd
@@ -38,7 +38,7 @@ COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/.local/share/bts/sparkd/da
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_symbols
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS="requirements.txt"
-ARG SPARKD_ENTRYPOINT
+ARG ENTRYPOINT_SCRIPT
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -51,8 +51,9 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
-COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
+COPY --chown=bts:bts "${ENTRYPOINT_SCRIPT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"
 # selected image
 FROM ${IMAGE}
 ENTRYPOINT "./sparkd-entrypoint.sh"
@@ -1,5 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/all_crypto
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/gateio_dev/requirements.txt
 SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
@@ -1,11 +0,0 @@
 {
  "currencies": [
    "USDT"
  ],
  "futures": [
    "BTC",
    "ETH",
    "SOL",
    "XRP"
  ]
 }
@@ -1,16 +0,0 @@
 {
  "currencies": [
    "USDT"
  ],
  "futures": [
    "BTC",
    "ETH",
    "SOL",
    "XRP"
  ],
  "use_testnet": false,
  "logging": {
    "console_level": "WARNING",
    "file_level": "INFO"
  }
 }
@@ -1,71 +0,0 @@
 PublisherOptions {
  value {
    network_interface: "0.0.0.0"
    heartbeat_ms: 5000
    heartbeat_boot_seconds: 60
    skip_options: true
    hot_initial: 1024
    hot_growth: 1024
    cold_initial: 8192
    cold_growth: 2048
    channel_prefix: ""
    disable_heartbeat: true
  }
 }
 Logging {
  value {
    initial_queue_size: 120001
    max_queue_growth: 4
    file {
      default_level: "Info"
      subsystem {
        name: "md"
        level: "Diag"
      }
    }
  }
 }
 MarketDataConfig {
  value {
    # python_md_providers {
    #   venue: VENUE_GATEIO
    #   module_path: "python_md_providers"
    #   provider_type: "gateio_md_provider.GateIoMdProvider"
    # }
    python_md_providers {
      venue: VENUE_BYBIT
      module_path: "python_interop_plugins"
      provider_type: "bybit_md_provider.BybitMdProvider"
    }
    binance_usds_futures {
      num_depth_levels: 5
      url_rest_base: "https://fapi.binance.com"
      url_ws_base: "wss://fstream.binance.com"
      security_criteria {
        clauses {
          security_types: SECURITY_TYPE_FUTURE
          symbols: @SymbolLists.list[Gateio].symbol
        }
      }
    }
     okx {
       num_depth_levels: 5
       url_rest_base: "https://www.okx.com"
       url_ws_base: "wss://ws.okx.com:8443"
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_FUTURE
           symbols: @SymbolLists.list[Gateio].symbol
         }
       }
       is_application_ping_pong_enabled: true
       application_ping_interval_ms: 20000
       application_pong_timeout_ms: 10000
       is_protocol_ping_pong_enabled: true
       protocol_ping_interval_ms: 60000
       protocol_pong_timeout_ms: 30000
     }
  }
 }
@@ -1,32 +0,0 @@
  SymbolLists {
  value {
    list {
      key: "Gateio"
      symbol: "SOL"
      symbol: "BTC"
      symbol: "ETH"
      symbol: "XRP"
    }
  }
 }
 Symbols {
  value {
    venues {
      name: "VENUE_GATEIO"
      spot: @SymbolLists.list[Gateio].symbol
    }
    venues {
      name: "VENUE_BYBIT"
      future: @SymbolLists.list[Gateio].symbol
    }
    venues {
      name: "VENUE_BINANCE_USDS_FUTURES"
      future: @SymbolLists.list[Gateio].symbol
    }
    venues {
      name: "VENUE_OKX"
      future: @SymbolLists.list[Gateio].symbol
    }
  }
 }
@@ -1,7 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_dev_replay_aws
 SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_dev_replay_aws/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
 SPARKD__SHM_SIZE=512m
 SPARKD__MEMORY=1g
@@ -1,61 +0,0 @@
 SymbolLists {
  value {
    list {
      key: "GlbxFutures"
      symbol: "ES"
    }
    # list {
    #   key: "GlbxOptions"
    #   symbol: "E1A"
    #   symbol: "E1B"
    #   symbol: "E1C"
    #   symbol: "E1D"
    #   symbol: "E2A"
    #   symbol: "E2B"
    #   symbol: "E2C"
    #   symbol: "E2D"
    #   symbol: "E3A"
    #   symbol: "E3B"
    #   symbol: "E3C"
    #   symbol: "E3D"
    #   symbol: "E4A"
    #   symbol: "E4B"
    #   symbol: "E4C"
    #   symbol: "E4D"
    #   symbol: "E5A"
    #   symbol: "E5B"
    #   symbol: "E5C"
    #   symbol: "E5D"
    #   symbol: "EW1"
    #   symbol: "EW2"
    #   symbol: "EW3"
    #   symbol: "EW4"
    #   symbol: "ESH9"
    #   symbol: "ESZ6"
    #   symbol: "ESH0"
    #   symbol: "ESM6"
    #   symbol: "ESZ7"
    #   symbol: "ESH6"
    #   symbol: "ESM7"
    #   symbol: "ESZ8"
    #   symbol: "ESH7"
    #   symbol: "ESH8"
    #   symbol: "ESU6"
    #   symbol: "ESZ5"
    #   symbol: "ESZ9"
    #   symbol: "E1D"
    #   symbol: "ESZ5"
    #   symbol: "ES"
    # }
  }
 }
 Symbols {
  value {
    venues {
        name: "VENUE_GLBX"
        future: @SymbolLists.list[GlbxFutures].symbol
        # option: @SymbolLists.list[GlbxOptions].symbol
    }
  }
 }
@@ -0,0 +1,4 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_replay
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_replay/requirements.txt
 SPARKD__ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
 TZ=America/Chicago
@@ -0,0 +1,38 @@
 PublisherOptions {
  value {
    network_interface: "0.0.0.0"
    heartbeat_ms: 5000
    heartbeat_boot_seconds: 60
    skip_options: true
    hot_initial: 1024
    hot_growth: 1024
    cold_initial: 8192
    cold_growth: 2048
    channel_prefix: ""
    disable_heartbeat: true
  }
 }
 MarketDataConfig {
  value {
   # glbx{
   #   use_databento: true
   #  databento_settings {
   #     read_from_file: true
   #    key: "db-DgxpuTXneNgVvd7sJCAEKYDhyCApW"
   #     bbo_file_name: "glbx-mdp3-20251103.mbp-1.dbn.zst"
   #     contracts_file_name: "glbx-mdp3-20251103.definition.dbn.zst"
   #   }
   # }
   # recording{
   #    venue: VENUE_GLBX
   #     num_of_seconds_to_run_file_writer: 240
   #     num_of_seconds_to_run_db_writer : 0
   #     n_book_levels: 10 
   #     batch_size: 100 #for the db writer
   # }
   replay {
      venue: VENUE_GLBX
    }
  }
 }
@@ -0,0 +1,48 @@
 Symbols {
  value {
    venues {
        name: "VENUE_GLBX"
        future: "ES"
        # option: "E1A"
        # option: "E1B"
        # option: "E1C"
        # option: "E1D"
        # option: "E2A"
        # option: "E2B"
        # option: "E2C"
        # option: "E2D"
        # option: "E3A"
        # option: "E3B"
        # option: "E3C"
        # option: "E3D"
        # option: "E4A"
        # option: "E4B"
        # option: "E4C"
        # option: "E4D"
        # option: "E5A"
        # option: "E5B"
        # option: "E5C"
        # option: "E5D"
        # option: "EW1"
        # option: "EW2"
        # option: "EW3"
        # option: "EW4"
        # option: "ESH9"
        # option: "ESZ6"
        # option: "ESH0"
        # option: "ESM6"
        # option: "ESZ7"
        # option: "ESH6"
        # option: "ESM7"
        # option: "ESZ8"
        # option: "ESH7"
        # option: "ESH8"
        # option: "ESU6"
        # option: "ESZ5"
        # option: "ESZ9"
        # option: "E1D"
        # option: "ESZ5"
        # option: "ES"
    }
  }
 }
@@ -1,10 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
 SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
 SPARKD__SHM_SIZE=8g
 SPARKD__MEMORY=16g
 SPARKD__CONFIG_PORT=9000
 SPARKD__QUICKBUS_PORT=9001
 SPARKD__SIM_FIX_PORT=9002
@@ -1,5 +0,0 @@
 numpy==2.2.6
 pybit==5.11.0
 requests==2.31.0
 websocket-client==1.8.0
 websockets==15.0.1
@@ -1,140 +0,0 @@
 DeploymentType {
  value {
    type: "dev"
  }
 }
 PublisherOptions {
   value {
      network_interface_a: 'ens65f1'
      network_interface_b: 'ens65f0'
      #these get overriden if not entered here
      heartbeat_ms: 5000
      heartbeat_boot_seconds: 60
      skip_options: false
      hot_initial: 1024
      hot_growth: 1024
      cold_initial: 8192
      cold_growth: 2048
      channel_prefix: ""
      disable_heartbeat: false
   }
 }
 CMEOptions {
  value {
    feed_engine {
      name: "mdp3feed"
      spin_time: 0
      channels_per_thread: 3
      data_wait: 10
      udp_socket_buffer_size: 268435456
    }
    channel {
      id: "*"
      feed_engine: "mdp3feed"
      use_tcp_replay: false
      recovery_queue_size: 640000
      lost_packet_wait_timeout_ms: 10
      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
      #max_gap_recovery_cache_size: 4
      use_natural_refresh: false
      join_snapshot_on_gap: false
    }
  }
 }
 Logging {
  value {
    initial_queue_size: 120001
    max_queue_growth: 4
    console {
      default_level: "Info"
      subsystem {
        name: "md"
        level: "Info"
      }
      subsystem {
        name: "OrderMan"
        level: "Info"
      }
    }
  }
 }
 MarketDataConfig {
  value {
    glbx {
       unique_check_value: 15
       use_databento: false
    }
   # recording {
   #    venue: VENUE_GLBX
   #    num_of_seconds_to_run_file_writer: 300
   #    num_of_seconds_to_run_db_writer : 0
   #    n_book_levels: 10 
   #    batch_size: 100 #for the db writer
   #    enable_book_recording: true
   #    enable_trade_recording:  true
   #    enable_bbo_recording: false
   #    enable_tbbo_recording:  false
   #    enable_misc_recording: true
   #}
 }
 }
 JetFIX {
  value {
    connection {
      name: "maw_004_64"
      venue: VENUE_GLBX
      socket_connect_host_ip: "69.50.112.177"
      socket_connect_port: 44040
      sender_compid: "MAW004"
      target_compid: "CME"
      target_subid: "64"
      cme_market_segment_id: 64
      start_time: "00:10:00"
      end_time: "23:55:00"
      start_day: "sunday"
      end_day: "friday"
      seconds_between_hbs: 15
      sender_subid: "BTS"
      seconds_to_pause: 2
      max_connection_attempts: 100
      use_ilink3: true
    }
  }
 }
 OrdermanCfg {
  value {
    executor {
      thread_name: "oman_orders"
      poll: true 
      hosted_actor: "MDSource"
      hosted_actor: "OrderManager"
      hosted_actor: "ExecutionGateway"
    }
    executor {
      thread_name: "oman_slow"
            poll: false 
      hosted_actor: "*"
    }
    fallback_cust_order_handling_inst: "Y"
  }
 }
 NotSimExecution {
  value {
    entries {
      venue: VENUE_GLBX
    }
  }
 }
 Ports {
   value {
      config_port: 9000
      qbus_port: 9001
      glbx_sim_fix_port: 9002
      ice_sim_fix_port: 9003
      replicator_port: 9080
   }
 }
@@ -0,0 +1,5 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
 SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
@@ -24,9 +24,38 @@ PublisherOptions {
  }
 }
 Logging {
  value {
    initial_queue_size: 120001
    max_queue_growth: 4
    console {
      default_level: "Info"
      subsystem {
        name: "md"
        level: "Warn"
      }
      subsystem {
        name: "OrderMan"
        level: "Info"
      }
    }
    file {
      default_level: "Info"
      subsystem {
        name: "OrderMan"
        level: "Diag"
      }
      subsystem {
        name: "Http"
        level: "Diag"
      }
    }
  }
 }
 MarketDataConfig {
  value {
-   replay {
+    replay {
      venue: VENUE_GLBX
    }
  }
@@ -0,0 +1,8 @@
 Symbols {
  value {
    venues {
      name: "VENUE_GLBX"
      future: "ES"
    }
  }
 }
@@ -1,7 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
 SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf
 SPARKD__SHM_SIZE=8g
 SPARKD__MEMORY=16g
@@ -1,5 +0,0 @@
 numpy==2.2.6
 pybit==5.11.0
 requests==2.31.0
 websocket-client==1.8.0
 websockets==15.0.1
@@ -1,152 +0,0 @@
 DeploymentType {
  value {
    type: "prod"
  }
 }
 PublisherOptions {
   value {
      network_interface_a: 'enp193s0f0np0'
      network_interface_b: 'enp193s0f1np1'
      #these get overriden if not entered here
      heartbeat_ms: 5000
      heartbeat_boot_seconds: 60
      skip_options: false
      hot_initial: 1024
      hot_growth: 1024
      cold_initial: 8192
      cold_growth: 2048
      channel_prefix: ""
      disable_heartbeat: false
   }
 }
 CMEOptions {
  value {
    feed_engine {
      name: "mdp3feed"
      spin_time: 0
      channels_per_thread: 3
      data_wait: 10
      udp_socket_buffer_size: 268435456
    }
    channel {
      id: "*"
      feed_engine: "mdp3feed"
      use_tcp_replay: false
      recovery_queue_size: 640000
      lost_packet_wait_timeout_ms: 10
      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
      #max_gap_recovery_cache_size: 4
      use_natural_refresh: false
      join_snapshot_on_gap: false
    }
  }
 }
 Logging {
  value {
    initial_queue_size: 120001
    max_queue_growth: 4
    console {
      default_level: "Info"
      subsystem {
        name: "md"
        level: "Info"
      }
      subsystem {
        name: "OrderMan"
        level: "Info"
      }
    }
  }
 }
 MarketDataConfig {
  value {
    glbx {
       unique_check_value: 15
       use_databento: false
    }
   # recording {
   #    venue: VENUE_GLBX
   #    num_of_seconds_to_run_file_writer: 300
   #    num_of_seconds_to_run_db_writer : 0
   #    n_book_levels: 10 
   #    batch_size: 100 #for the db writer
   #    enable_book_recording: true
   #    enable_trade_recording:  true
   #    enable_bbo_recording: false
   #    enable_tbbo_recording:  false
   #    enable_misc_recording: true
   #}
 }
 }
 JetFIX {
  value {
    connection {
      name: "u6p_7qb_54"
      venue: VENUE_GLBX
      socket_connect_host_ip: "205.209.201.97"
      socket_connect_port: 10979
      sender_compid: "U6P7QB"
      target_compid: "CME"
      target_subid: "54"
      cme_market_segment_id: 54
      start_time: "00:10:00"
      end_time: "23:55:00"
      start_day: "sunday"
      end_day: "friday"
      seconds_between_hbs: 15
      sender_subid: "BTS"
      seconds_to_pause: 1
      max_order_msgs_per_window: 300
      max_connection_attempts: 2
      use_ilink3: true
    }
   connection {
      name: "u6p_7qb_64"
      venue: VENUE_GLBX
      socket_connect_host_ip: "205.209.201.1"
      socket_connect_port: 10979
      sender_compid: "U6P7QB"
      target_compid: "CME"
      target_subid: "64"
      cme_market_segment_id: 64
      start_time: "00:10:00"
      end_time: "23:55:00"
      start_day: "sunday"
      end_day: "friday"
      seconds_between_hbs: 15
      sender_subid: "BTS"
      seconds_to_pause: 1
      max_connection_attempts: 2
      max_order_msgs_per_window: 300
      use_ilink3: true
    }
  }
 }
 OrdermanCfg {
  value {
    executor {
      thread_name: "oman_orders"
      poll: true 
      hosted_actor: "MDSource"
      hosted_actor: "OrderManager"
      hosted_actor: "ExecutionGateway"
    }
    executor {
      thread_name: "oman_slow"
            poll: false 
      hosted_actor: "*"
    }
    fallback_cust_order_handling_inst: "Y"
  }
 }
 NotSimExecution {
  value {
    entries {
      venue: VENUE_GLBX
    }
  }
 }
@@ -1,61 +0,0 @@
 SymbolLists {
  value {
    list {
      key: "GlbxFutures"
      symbol: "ES"
    }
    list {
      key: "GlbxOptions"
      symbol: "E1A"
      symbol: "E1B"
      symbol: "E1C"
      symbol: "E1D"
      symbol: "E2A"
      symbol: "E2B"
      symbol: "E2C"
      symbol: "E2D"
      symbol: "E3A"
      symbol: "E3B"
      symbol: "E3C"
      symbol: "E3D"
      symbol: "E4A"
      symbol: "E4B"
      symbol: "E4C"
      symbol: "E4D"
      symbol: "E5A"
      symbol: "E5B"
      symbol: "E5C"
      symbol: "E5D"
      symbol: "EW1"
      symbol: "EW2"
      symbol: "EW3"
      symbol: "EW4"
      symbol: "ESH9"
      symbol: "ESZ6"
      symbol: "ESH0"
      symbol: "ESM6"
      symbol: "ESZ7"
      symbol: "ESH6"
      symbol: "ESM7"
      symbol: "ESZ8"
      symbol: "ESH7"
      symbol: "ESH8"
      symbol: "ESU6"
      symbol: "ESZ5"
      symbol: "ESZ9"
      symbol: "E1D"
      symbol: "ESZ5"
      symbol: "ES"
    }
  }
 }
 Symbols {
  value {
    venues {
        name: "VENUE_GLBX"
        future: @SymbolLists.list[GlbxFutures].symbol
        option: @SymbolLists.list[GlbxOptions].symbol
    }
  }
 }
@@ -1,7 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
 SPARKD__DATA_DIR=/home/dk/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/dk/8000/sparkdata/conf
 SPARKD__SHM_SIZE=8g
 SPARKD__MEMORY=16g
@@ -1,5 +0,0 @@
 numpy==2.2.6
 pybit==5.11.0
 requests==2.31.0
 websocket-client==1.8.0
 websockets==15.0.1
@@ -1,101 +0,0 @@
 DeploymentType {
  value {
    type: "prod"
  }
 }
 PublisherOptions {
   value {
      network_interface_a: 'enp193s0f0np0'
      network_interface_b: 'enp193s0f1np1'
      #these get overriden if not entered here
      heartbeat_ms: 5000
      heartbeat_boot_seconds: 60
      skip_options: false
      hot_initial: 1024
      hot_growth: 1024
      cold_initial: 8192
      cold_growth: 2048
      channel_prefix: ""
      disable_heartbeat: false
   }
 }
 CMEOptions {
  value {
    feed_engine {
      name: "mdp3feed"
      spin_time: 0
      channels_per_thread: 3
      data_wait: 10
      udp_socket_buffer_size: 268435456
    }
    channel {
      id: "*"
      feed_engine: "mdp3feed"
      use_tcp_replay: false
      recovery_queue_size: 640000
      lost_packet_wait_timeout_ms: 10
      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
      #max_gap_recovery_cache_size: 4
      use_natural_refresh: false
      join_snapshot_on_gap: false
    }
  }
 }
 Logging {
  value {
    initial_queue_size: 120001
    max_queue_growth: 4
    console {
      default_level: "Info"
      subsystem {
        name: "md"
        level: "Info"
      }
      subsystem {
        name: "OrderMan"
        level: "Info"
      }
    }
  }
 }
 MarketDataConfig {
  value {
    glbx {
       unique_check_value: 15
       use_databento: false
    }
   # recording {
   #    venue: VENUE_GLBX
   #    num_of_seconds_to_run_file_writer: 300
   #    num_of_seconds_to_run_db_writer : 0
   #    n_book_levels: 10 
   #    batch_size: 100 #for the db writer
   #    enable_book_recording: true
   #    enable_trade_recording:  true
   #    enable_bbo_recording: false
   #    enable_tbbo_recording:  false
   #    enable_misc_recording: true
   #}
 }
 }
 OrdermanCfg {
  value {
    executor {
      thread_name: "oman_orders"
      poll: true 
      hosted_actor: "MDSource"
      hosted_actor: "OrderManager"
      hosted_actor: "ExecutionGateway"
    }
    executor {
      thread_name: "oman_slow"
            poll: false 
      hosted_actor: "*"
    }
    fallback_cust_order_handling_inst: "Y"
  }
 }
@@ -1,61 +0,0 @@
 SymbolLists {
  value {
    list {
      key: "GlbxFutures"
      symbol: "ES"
    }
    list {
      key: "GlbxOptions"
      symbol: "E1A"
      symbol: "E1B"
      symbol: "E1C"
      symbol: "E1D"
      symbol: "E2A"
      symbol: "E2B"
      symbol: "E2C"
      symbol: "E2D"
      symbol: "E3A"
      symbol: "E3B"
      symbol: "E3C"
      symbol: "E3D"
      symbol: "E4A"
      symbol: "E4B"
      symbol: "E4C"
      symbol: "E4D"
      symbol: "E5A"
      symbol: "E5B"
      symbol: "E5C"
      symbol: "E5D"
      symbol: "EW1"
      symbol: "EW2"
      symbol: "EW3"
      symbol: "EW4"
      symbol: "ESH9"
      symbol: "ESZ6"
      symbol: "ESH0"
      symbol: "ESM6"
      symbol: "ESZ7"
      symbol: "ESH6"
      symbol: "ESM7"
      symbol: "ESZ8"
      symbol: "ESH7"
      symbol: "ESH8"
      symbol: "ESU6"
      symbol: "ESZ5"
      symbol: "ESZ9"
      symbol: "E1D"
      symbol: "ESZ5"
      symbol: "ES"
    }
  }
 }
 Symbols {
  value {
    venues {
        name: "VENUE_GLBX"
        future: @SymbolLists.list[GlbxFutures].symbol
        option: @SymbolLists.list[GlbxOptions].symbol
    }
  }
 }
@@ -0,0 +1,5 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
 SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf
@@ -1,12 +1,18 @@
-DeploymentType {
+JetFIX {
  value {
-    type: "prod"
+    connection {
      name: "glbx_1"
      venue: VENUE_GLBX
      max_order_msgs_per_window: 300
      window_duration_msecs: 1000
    }
  }
 }
 PublisherOptions {
   value {
-      network_interface_a: 'ens5'
+      network_interface_a: 'ens3f0np0'
-      network_interface_b: 'ens5'
+      network_interface_b: 'ens3f1np1'
      #these get overriden if not entered here
      heartbeat_ms: 5000
      heartbeat_boot_seconds: 60
@@ -35,7 +41,7 @@ CMEOptions {
      use_tcp_replay: false
      recovery_queue_size: 640000
      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
      #max_gap_recovery_cache_size: 4
      use_natural_refresh: false
      join_snapshot_on_gap: false
@@ -67,17 +73,32 @@ MarketDataConfig {
       unique_check_value: 15
       use_databento: false
    }
-   # recording {
+  }
   #    venue: VENUE_GLBX
   #    num_of_seconds_to_run_file_writer: 300
   #    num_of_seconds_to_run_db_writer : 0
   #    n_book_levels: 10 
   #    batch_size: 100 #for the db writer
   #    enable_book_recording: true
   #    enable_trade_recording:  true
   #    enable_bbo_recording: false
   #    enable_tbbo_recording:  false
   #    enable_misc_recording: true
   #}
 }
 }
 # JetFIX {
 #    value {
 #     #connection: @JetFIXSimConnection.sim
 #     connection {
 #     name: "mlx_004_54"
 #     venue: VENUE_GLBX
 #     socket_connect_host_ip: "69.50.112.191"
 #     socket_connect_port: 42876
 #     sender_compid: "MLX004"
 #     target_compid: "CME"
 #     target_subid: "54"
 #     cme_market_segment_id: 54
 #     start_time: "00:10:00"
 #     end_time: "23:55:00"
 #     start_day: "sunday"
 #     end_day: "friday"
 #     seconds_between_hbs: 30
 #     sender_subid: "BTS"
 #     use_ilink3: true
 #   #   raw_data: "S4PMA"
 #   #   auto_fix_seq_num: true
 #   }
 #    }
 # }
@@ -1,4 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay/requirements.txt
 TZ=America/Chicago
 SPARKD__MD_RECORDING_DIR=/home/wedbushprod/8000/md_recording
@@ -1,46 +0,0 @@
 SymbolLists {
  value {
    list {
      key: "GlbxFutures"
      symbol: "ES"
    }
    list {
      key: "GlbxOptions"
      symbol: "E1A"
      symbol: "E1B"
      symbol: "E1C"
      symbol: "E1D"
      symbol: "E2A"
      symbol: "E2B"
      symbol: "E2C"
      symbol: "E2D"
      symbol: "E3A"
      symbol: "E3B"
      symbol: "E3C"
      symbol: "E3D"
      symbol: "E4A"
      symbol: "E4B"
      symbol: "E4C"
      symbol: "E4D"
      symbol: "E5A"
      symbol: "E5B"
      symbol: "E5C"
      symbol: "E5D"
      symbol: "ES"
      symbol: "EW1"
      symbol: "EW2"
      symbol: "EW3"
      symbol: "EW4"
    }
  }
 }
 Symbols {
  value {
    venues {
        name: "VENUE_GLBX"
        future: @SymbolLists.list[GlbxFutures].symbol
        option: @SymbolLists.list[GlbxOptions].symbol
    }
  }
 }
@@ -1,9 +0,0 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay_wb
 SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
 SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
 SPARKD__MD_RECORDING_DIR=/home/wedbushprod/md_recording
 SPARKD__CONFIG_PORT=9000
 SPARKD__QUICKBUS_PORT=9001
 SPARKD__SIM_FIX_PORT=9002
@@ -1,5 +0,0 @@
 numpy==2.2.6
 pybit==5.11.0
 requests==2.31.0
 websocket-client==1.8.0
 websockets==15.0.1
@@ -1,46 +0,0 @@
 JetFIX {
  value {
    connection {
      name: "glbx_1"
      venue: VENUE_GLBX
      max_order_msgs_per_window: 300
      window_duration_msecs: 1000
    }
  }
 }
 Ports {
   value {
      config_port: 9000
      qbus_port: 9001
      glbx_sim_fix_port: 9002
      ice_sim_fix_port: 9003
   }
 }
 Logging {
  value {
    initial_queue_size: 120001
    max_queue_growth: 4
    console {
      default_level: "Info"
      subsystem {
        name: "md"
        level: "Info"
      }
      subsystem {
        name: "OrderMan"
        level: "Info"
      }
    }
  }
 }
 MarketDataConfig {
  value {
     replay {
      venue: VENUE_GLBX
    }
  }
 }
@@ -1,61 +0,0 @@
 SymbolLists {
  value {
    list {
      key: "GlbxFutures"
      symbol: "ES"
    }
    list {
      key: "GlbxOptions"
      symbol: "E1A"
      symbol: "E1B"
      symbol: "E1C"
      symbol: "E1D"
      symbol: "E2A"
      symbol: "E2B"
      symbol: "E2C"
      symbol: "E2D"
      symbol: "E3A"
      symbol: "E3B"
      symbol: "E3C"
      symbol: "E3D"
      symbol: "E4A"
      symbol: "E4B"
      symbol: "E4C"
      symbol: "E4D"
      symbol: "E5A"
      symbol: "E5B"
      symbol: "E5C"
      symbol: "E5D"
      symbol: "EW1"
      symbol: "EW2"
      symbol: "EW3"
      symbol: "EW4"
      symbol: "ESH9"
      symbol: "ESZ6"
      symbol: "ESH0"
      symbol: "ESM6"
      symbol: "ESZ7"
      symbol: "ESH6"
      symbol: "ESM7"
      symbol: "ESZ8"
      symbol: "ESH7"
      symbol: "ESH8"
      symbol: "ESU6"
      symbol: "ESZ5"
      symbol: "ESZ9"
      symbol: "E1D"
      symbol: "ESZ5"
      symbol: "ES"
    }
  }
 }
 Symbols {
  value {
    venues {
        name: "VENUE_GLBX"
        future: @SymbolLists.list[GlbxFutures].symbol
        option: @SymbolLists.list[GlbxOptions].symbol
    }
  }
 }
@@ -4,12 +4,9 @@ init_timezone() {
  tz="${1:-$TZ}"
  zoneinfo_dir=/usr/share/zoneinfo
  if [ -f "$zoneinfo_dir/$tz" -a "$(readlink -f /etc/localtime)" != "$zoneinfo_dir/$tz" ]; then
-    echo "Setting locales and timezone to $tz"
+    echo "Initializing locales and timezone"
-    if [ -e /etc/localtime ]; then
+    ln -snf "$zoneinfo_dir/$tz" /etc/localtime
-      sudo -n rm -f /etc/localtime
+    sudo echo $tz > /etc/timezone
    fi
    sudo -n ln -snf "$zoneinfo_dir/$tz" /etc/localtime
    echo $tz | sudo -n tee /etc/timezone > /dev/null
  fi
 }
@@ -39,17 +36,10 @@ roll_log() {
  fi
 }
 # Acquire sudo permission once for the script (needed for a few calls)
 echo bts | sudo -S -v
 data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
-sudo -n chown -R bts:bts "$data_root"
+echo bts | sudo -S chown -R bts:bts "$data_root"
 load_env "$data_root/conf/.env.local"
 init_timezone "$TZ"
 roll_log "$data_root/data/logging.sparkd/log"
 symlink "$data_root/conf/plugins" /home/bts/bin/plugins
 if  [ -e "/md_recording" -a ! -e "/home/bts/.local/share/bts/sparkd/data/md_recording" ]; then
  symlink "/md_recording" /home/bts/.local/share/bts/sparkd/data/md_recording
 fi
 bash -c "${@:-./sparkd run}"
Author	SHA1	Message	Date
John Colvin	bef84d6dc3	build: Install boost via vcpkg on linux, update ccapi Old ccapi wasn't compatible with new boost	2026-03-20 17:02:44 +00:00
John Colvin	b061509126	build: Compile spark-windows with clang Made a couple minor changes to cpp to address new werrors	2026-03-20 17:02:44 +00:00
John Colvin	5c0487358e	build: Install boost via vcpkg on windows	2026-03-20 17:02:44 +00:00
John Colvin	5c7eedbf7b	build: Don't activate venv when generating individual protos We activate the venv for the whole build	2026-03-20 17:02:44 +00:00
John Colvin	74475cb68f	deploy: Resolve profile dirs with regex patterns	2026-03-20 17:02:44 +00:00
John Colvin	24978f76b5	improve: Wait until py work requested before loading py modules So we don't crash due to missing py modules when no py plugins are running.	2026-03-20 17:02:44 +00:00
John Colvin	0698567f53	deploy: Add sparkd-shell service to compose.yaml for troubleshooting	2026-03-20 17:02:44 +00:00
John Colvin	ec78e35733	cleanup: Remove unused code	2026-03-20 17:02:44 +00:00
John Colvin	4801ce6b83	refactor: Wrap md run logic in class MarketDataSession Start in the constructor, gracefully exit in the destructor.	2026-03-20 17:02:44 +00:00
John Colvin	7eb8cd5ead	Allow md recording dir to be mounted as volume in docker compose	2026-03-20 17:02:44 +00:00
John Colvin	d7b42e3cee	refactor: Split MDRelay and QueryServer up into smaller files This code is inherently complicated. It's hard to follow when everything is in one giant file.	2026-03-20 17:02:44 +00:00
John Colvin	35d65f107b	deploy: Configure timezone in entrypoint script	2026-03-20 17:02:44 +00:00
John Colvin	e335a2d827	deploy: Move compose entrypoint to dedicated script	2026-03-20 17:02:44 +00:00
John Colvin	abc7d44ce2	deploy: Fix typo in sparkd conf dir path in compose.yaml	2026-03-20 17:02:44 +00:00
John Colvin	85ff7f412a	cleanup: Fix a few more build warnings	2026-03-20 17:02:44 +00:00
John Colvin	8a3ad08456	refactor: Replace DataPacket with IDataPacket in more places	2026-03-20 17:02:44 +00:00
John Colvin	f181d34cbe	build: Check if venv already active before activating	2026-03-20 17:02:44 +00:00
John Colvin	002087f62a	refactor: Replace TableFormatter with table.h Also move to "format" dir	2026-03-20 17:02:44 +00:00
John Colvin	e37ae056c0	refactor: Replace StringFinder/Joiner.h with find/join.h Also move to "algorithm" dir	2026-03-20 17:02:44 +00:00
John Colvin	b8a7f398c9	refactor: Rename a few TitleCase .h files snake_case * EnumMacros.h -> enum_macros.h * TimeSpan.h ---> time_span.h * TimeStamp.h --> time_stamp.h	2026-03-20 17:02:44 +00:00
John Colvin	2038d747d8	diag: Use structured logs in MDP3Listener::onOption	2026-03-20 17:02:44 +00:00
John Colvin	edf127069f	deploy: Install tzdata in linux_dotnet8 Dockerfile So we can configure timezone in compose services. Specifically, we need this to support live cme md (so onixs doesn't think messages are "too old").	2026-03-20 17:02:44 +00:00
John Colvin	07448cede9	cleanup: Remove unused code	2026-03-20 17:02:44 +00:00
John Colvin	9aef42cb21	refactor: Add new glbx variable ticks to security (shm and proto)	2026-03-20 17:02:44 +00:00
John Colvin	baa43cbecf	refactor: Enable nullability in Bts.Spark.Api	2026-03-08 16:00:50 +00:00
John Colvin	de3e491843	build: Upgrade cs LangVersion 11 -> 12	2026-03-08 16:00:50 +00:00
John Colvin	7cc09350a0	refactor: Rename SparkApiCredentials -> SparkCredentials	2026-03-08 16:00:50 +00:00
John Colvin	059e1202e1	refactor: Rename SparkdLoginService -> SparkLoginService This makes it consistent with other type names (we don't use "Sparkd" anywhere else).	2026-03-08 16:00:50 +00:00
John Colvin	9093f56498	refactor: Rename SparkdCredentials.cs -> SparkApiCredentials.cs Matches the class name	2026-03-08 16:00:50 +00:00
John Colvin	649b4560ff	cleanup: Remove unnecessary subscription wrapper in cs sdk	2026-03-08 16:00:50 +00:00
John Colvin	6c060f6171	cleanup: Fix indentation in TopStream Whitespace changes	2026-03-08 16:00:50 +00:00
John Colvin	581b9259b8	cleanup: Fix build warnings in EqualityComparer Nullability issues	2026-03-08 16:00:50 +00:00
John Colvin	e7bd57964e	refactor: Rename Optional members PascalCase Also reformat code according to new style guidlines	2026-03-08 16:00:50 +00:00
John Colvin	6ea955aacd	refactor: Replace ToDictionaryCrushed/Dupes w/ DuplicateKeyPolicy enum Overload ToDictionary with this enum instead.	2026-03-08 16:00:49 +00:00
John Colvin	29d028fee2	refactor: Replace find_first_index with FindIndex	2026-03-08 16:00:49 +00:00
John Colvin	dc5da87a65	cleanup: Fix cs build warnings	2026-03-08 16:00:49 +00:00
John Colvin	7de98a313f	refactor: Break up Bts.Spark.Core into a few diff projs This is in anticipation of decoupling BlueOrb from Spark. * Bts.Collections * Bts.Core * Bts.Disposables * Bts.Numerics	2026-03-08 16:00:49 +00:00
John Colvin	7873e76ba0	build: Clean up spark_client.sln * Remove unused csprojs related to old sdk * Remove erroneous build type config	2026-03-08 16:00:49 +00:00