fix: build was broken, spark.dll being used by another process

Native assets (libspark.dll/so) are copied to the outdir for local
builds, and packed in nupkg files for sdk releases. The dest path in the
package is different from the path in the outdir, so we configured
Copy/Pack as separate msbuild items. However, both had the same
Include="@(NativeWindows/LinuxAsset)", so Pack="false" in the Copy item
was overriding Pack="true" in the Pack item.

.env

@@ -51,7 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
-SPARKD__ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development

.vscode/launch.json

@@ -183,11 +183,10 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -214,7 +213,7 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
+      "preLaunchTask": "Build Spark Sdk (Debug)",
       "windows": {
         "program": "Bts.Spark.Sdk.Demo.exe",
       },
@@ -449,14 +448,15 @@
       "default": "Local",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
-      "description": "PubSub Configuration",
+      "description": "PubSubServer Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sdkDemoEnvironment",

Bts.Spark.Sdk.sln

@@ -17,14 +17,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{3BF4DF96-5AFF-4674-866A-71A8090627FF}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{62067659-AA2E-4CB5-B7CE-0B38EAA92080}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -95,54 +87,6 @@ Global
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.Build.0 = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.Build.0 = Debug|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.ActiveCfg = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.Build.0 = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.ActiveCfg = Release|Any CPU
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.Build.0 = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.Build.0 = Debug|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.Build.0 = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.ActiveCfg = Release|Any CPU
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.Build.0 = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.Build.0 = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.Build.0 = Debug|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.Build.0 = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.ActiveCfg = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.Build.0 = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.ActiveCfg = Release|Any CPU
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.Build.0 = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.Build.0 = Debug|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.Build.0 = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.ActiveCfg = Release|Any CPU
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -154,9 +98,5 @@ Global
 		{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{3BF4DF96-5AFF-4674-866A-71A8090627FF} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{62067659-AA2E-4CB5-B7CE-0B38EAA92080} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal

CMakePresets.json

@@ -32,20 +32,14 @@
         "lhs": "${hostSystemName}",
         "rhs": "Windows"
       },
-      "generator": "Ninja Multi-Config",
+      "generator": "Visual Studio 17 2022",
       "environment": {
-        "SPARK_CMAKE_CXX_FLAGS": "/WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
+        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "clang-cl",
-        "CMAKE_CXX_COMPILER": "clang-cl",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": {
-          "type": "BOOL",
-          "value": "ON"
-        },
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows"

Spark.Api.Server.sln

@@ -51,14 +51,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{802ECF0B-7264-4168-95D1-158E47FB65CC}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{0354655D-2394-42BB-B1D4-2D75B9BCABF1}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -579,102 +571,6 @@ Global
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.ActiveCfg = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.Build.0 = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.ActiveCfg = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.Build.0 = Release|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.ActiveCfg = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.Build.0 = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.Build.0 = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.ActiveCfg = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.Build.0 = Release|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.ActiveCfg = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.Build.0 = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.Build.0 = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.ActiveCfg = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.Build.0 = Release|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.ActiveCfg = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.Build.0 = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.Build.0 = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.ActiveCfg = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.Build.0 = Release|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -696,10 +592,6 @@ Global
 		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{C42F4CCB-9E65-4370-939A-527ACDBC4DA6} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{802ECF0B-7264-4168-95D1-158E47FB65CC} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0} = {385C4948-E080-56A6-E29E-8B0F4913175C}
-		{0354655D-2394-42BB-B1D4-2D75B9BCABF1} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}

compose.yaml

@@ -10,7 +10,7 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - ENTRYPOINT_SCRIPT=${SPARKD__ENTRYPOINT}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     env_file:
       - .env
@@ -19,6 +19,7 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       resources:
         limits:
@@ -35,7 +36,7 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - ENTRYPOINT_SCRIPT=${SPARKD__ENTRYPOINT}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     command: "do sleep 1; done"
     env_file:
@@ -53,6 +54,7 @@ services:
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
       - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
@@ -61,7 +63,7 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - ENTRYPOINT_SCRIPT=${SPARKD__ENTRYPOINT}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
     env_file:
       - .env
@@ -70,6 +72,7 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -77,7 +80,65 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
+  sparkd-multicast:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
+  
+  sparkd-multicast-cert:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -19,6 +19,11 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
   && apt-get install -y dotnet8
 
+RUN apt-get update && apt-get install -y gdb
+RUN apt-get update && apt-get install -y fish
+RUN apt-get update && apt-get install -y vim
+RUN apt-get update && apt-get install -y less
+
 RUN locale-gen en_US.UTF-8
 
 ARG docker_user="bts"

spark_client.sln

@@ -49,14 +49,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{00D21F82-61F4-467F-B125-268018A185E5}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{513A3F0D-46B5-4AC2-B700-8671933F6347}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{D419D996-9870-4FF5-82AD-08370D1C945F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -283,102 +275,6 @@ Global
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.ActiveCfg = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.Build.0 = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.ActiveCfg = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.Build.0 = Release|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.ActiveCfg = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.Build.0 = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.Build.0 = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.ActiveCfg = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.Build.0 = Release|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.ActiveCfg = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.Build.0 = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.Build.0 = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.ActiveCfg = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.Build.0 = Release|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.ActiveCfg = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.Build.0 = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.Build.0 = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.ActiveCfg = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.Build.0 = Release|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -406,9 +302,5 @@ Global
 		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
 		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
-		{00D21F82-61F4-467F-B125-268018A185E5} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{513A3F0D-46B5-4AC2-B700-8671933F6347} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{D419D996-9870-4FF5-82AD-08370D1C945F} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
-		{697BC3A3-1A98-43D3-A6E8-BB39A05EA049} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs

@@ -102,10 +102,10 @@ namespace BCG
          List<ColumnInfo> col_info = new List<ColumnInfo>();
          col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
          foreach (Pair<string, Type> p in group_by_col_info)
-            col_info.Add(new ColumnInfo(p.first ?? string.Empty, col_index++, ExceptionAssert.NotNull(p.second)));
+            col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
          foreach (string tc_prefix in tc_prefixes)
             foreach (Pair<string, Type> p in _cols_per_transposed)
-               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, ExceptionAssert.NotNull(p.second)));
+               col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
          IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
          if (_schema == null)
             _schema = schema;
@@ -182,7 +182,7 @@ namespace BCG
          if (_transpsed_col_value_comparer_wrapper != null)
             tc_values.Sort(_transpsed_col_value_comparer_wrapper);
          List<string> prefixes = new List<string>();
-         foreach (Pair<int, object?[]> p in tc_values)
+         foreach (Pair<int, object[]> p in tc_values)
             prefixes.Add(tc_prefixes[p.first]);
          _set_schema(prefixes, group_by_col_info, table_name);
          BlueTable result = new BlueTable(_schema);

src/DataStreamSuite/DataStream/Primitives/ConcatenateColumns/ConcatenateColumnsMapper.cs

@@ -1,17 +1,17 @@
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
-using System.Diagnostics.CodeAnalysis;
 using System.Text;
 
+using BCG;
+using BCG.Utilities;
+
 namespace BCG
 {
    public class ConcatenateColumnsMapper : IStreamMapper
    {
       protected IDictionary myTempColMap = new SortedList ();
-      protected ArrayList? myColMap;
-      protected BlueTableSchema? mySchema;
+      protected ArrayList myColMap;
+      protected BlueTableSchema mySchema;
 
       public ConcatenateColumnsMapper(params string [] columnDefs)
       {
@@ -55,7 +55,11 @@ namespace BCG
             myColMap = new ArrayList (myTempColMap.Count);
             foreach (DictionaryEntry de in myTempColMap)
             {
-               myColMap.Add(Array.ConvertAll((string[])de.Value!, c => inputSchema[c].colNo));
+               string[] cols = de.Value as string[];
+               ArrayList temp = new ArrayList ();
+               foreach (string col in cols)
+                  temp.Add (inputSchema[col].colNo);
+               myColMap.Add (temp.ToArray(typeof(int)) as int[]);
             }
             myTempColMap.Clear ();
          }
@@ -68,8 +72,6 @@ namespace BCG
 
       public virtual IBlueRow map (IBlueRow row)
       {
-         requireSchema();
-         requireColMap();
          object[] retVal = new object[mySchema.columnInfo.Length];
          row.CopyTo (retVal, myColMap.Count);
          for (int colNo = 0; colNo < myColMap.Count; colNo ++)
@@ -88,21 +90,11 @@ namespace BCG
       /// <returns></returns>
       protected virtual object myGetConcatValue (IBlueRow row, int colNo)
       {
-         requireColMap();
          StringBuilder b = new StringBuilder();
-         foreach (int cn in (int[])myColMap[colNo]!) {
-            b.Append(row[cn]?.ToString());
-         }
+         foreach (int cn in myColMap[colNo] as int[])
+            b.Append(row[cn].ToString());
          return b.ToString ();
       }
-
-      [MemberNotNull(nameof(myColMap))]
-      protected void requireColMap() =>
-         ExceptionAssert.NotNull(myColMap);
-
-      [MemberNotNull(nameof(mySchema))]
-      void requireSchema() =>
-         ExceptionAssert.NotNull(mySchema);
    }
 
    public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper 
@@ -115,21 +107,23 @@ namespace BCG
          myDicts = new HashTableND [columnDefs.Length];
          int i = 0;
          foreach (DictionaryEntry de in myTempColMap)
-            myDicts[i++] = new HashTableND(((string[])de.Value!).Length);
+            myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
       }
 
       protected override object myGetConcatValue (IBlueRow row, int colNo)
       {
-         requireColMap();
+         int[] cols = myColMap[colNo] as int[];
+         object[] vals = new object[cols.Length];
+         for (int i = 0; i < vals.Length; i++)
+            vals[i] = row[cols[i]];
          HashTableND ht = myDicts[colNo];
-         var vals = Array.ConvertAll((int[])myColMap[colNo]!, c => row[c]);
-         var rv = ht.getValue(vals);
+         object rv = ht.getValue (vals);
          if (rv != null)
             return rv;
          else
          {
             rv = base.myGetConcatValue (row, colNo);
-            var args = new object?[vals.Length + 1];
+            object[] args = new object[vals.Length + 1];
             vals.CopyTo (args, 0);
             args[args.Length-1] = rv;
             ht.add (args);

src/DataStreamSuite/DataStream/Primitives/Crushing/AverageCollectorStream.cs

@@ -20,7 +20,7 @@ namespace BCG
         {
             IDictionary<string, double> columnSums = new Dictionary<string, double>();
 
-            IBlueRow? resultRow = null;
+            IBlueRow resultRow = null;
             foreach (IBlueRow row in rows)
             {
                 resultRow = row;
@@ -29,7 +29,7 @@ namespace BCG
                 {
                     double sum = 0.0f;
                     columnSums.TryGetValue(colName, out sum);
-                    sum += (double)row[colName]!;
+                    sum += (double) row[colName];
                     columnSums[colName] = sum;
                 }
             }

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -1,3 +1,8 @@
+using System;
+using System.Collections;
+
+using BCG.Utilities;
+
 namespace BCG
 {
 	public class DistinctRowEvaluator : GroupByEvaluator

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -67,7 +67,7 @@ namespace BCG {
       }
       #endregion
 
-      public event FilterChangeDelegate? onFilterChange;
+      public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
       public bool filter(IBlueRow row) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -43,7 +43,7 @@ namespace bts.SilverPine {
          operations = operations_;
       }
 
-      public static IBlueRow? group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
          if (rows.Count() == 0) {
             return null;
          }
@@ -73,7 +73,7 @@ namespace bts.SilverPine {
       // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
       // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
 
-      public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
          if (rolling_rows.Count() == 0) {
             return null;
          }
@@ -123,4 +123,4 @@ namespace bts.SilverPine {
       }
 
    }
-}
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -1,4 +1,4 @@
-using Bts.DataStreamCore;
+using System;
 
 namespace BCG
 {
@@ -9,18 +9,18 @@ namespace BCG
 	public abstract class GroupByEvaluator : IGroupByEvaluator
 	{
       protected string[] myGroupByCols;
-      protected string? myOutputColumn;
-      protected GroupByOutputTable? myOutputTable;
-      protected string? myColMapExpression;
+      protected string myOutputColumn;
+      protected GroupByOutputTable myOutputTable;
+      protected string myColMapExpression;
 
       #region constructors
-		public GroupByEvaluator (string? outputCol, string[] groupByCols)
+		public GroupByEvaluator (string outputCol, string[] groupByCols)
 		{
          myOutputColumn = outputCol;
          myGroupByCols = groupByCols;
 		}
 
-      public GroupByEvaluator (string colMapExpression, string? outputCol, string[] groupByCols)
+      public GroupByEvaluator (string colMapExpression, string outputCol, string[] groupByCols)
          : this (outputCol, groupByCols)  {myColMapExpression = colMapExpression;}
       #endregion
 
@@ -52,20 +52,21 @@ namespace BCG
       /// <param name="r"></param>
       /// <param name="id"></param>
       /// <returns></returns>
-      protected IBlueRow? myProject(IBlueRow? r, int id) =>
-         r is null
-            ? null
-            : myColMapExpression is null || myColMapExpression.Length < 1
-               ? BlueDataUtility.getNewReplacedRow(r, id, "id")
-               : BlueDataUtility.getNewReplacedRow(myProjectRow(r, myColMapExpression), id, "id");
-
+      protected IBlueRow myProject (IBlueRow r, int id)
+      {
+         return r == null ? null :
+            myColMapExpression == null || myColMapExpression.Length < 1 ?
+               BlueDataUtility.getNewReplacedRow (r, id, "id") :
+               BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression),
+            id, "id");
+      }
+      
       /// <summary>
       /// Projects the id, conserving the id
       /// </summary>
       /// <param name="r"></param>
       /// <returns></returns>
-      protected IBlueRow? myProject(IBlueRow? r) =>
-         myProject(r, r is null ? int.MinValue : r.Id());
+      protected IBlueRow myProject (IBlueRow r)   {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
       #endregion
 
       IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs

@@ -1,6 +1,5 @@
-using Bts.DataStreamCore;
-
 using System;
+using BCG.Utilities;
 using System.Collections;
 
 namespace BCG
@@ -23,12 +22,10 @@ namespace BCG
       /// <param name="groupRepString"></param>
       public void addRow (IBlueRow row, string groupRepString)
       {
-         var grp = myDict[groupRepString];
-         if (grp is not null) {
-            myTable.deleteRow((IBlueRow)grp);
-         }
+         if (myDict[groupRepString] != null)
+            myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
          myTable.addRow (row);
-         myDict[groupRepString] = myTable[row.Id()];
+         myDict[groupRepString] = myTable[(int)row["id"]];
       }
 
       /// <summary>
@@ -38,7 +35,7 @@ namespace BCG
       public void removeRow (string groupRepString)
       {
          // BlueEventLog.logAssert (myDict[groupRepString] != null);
-         myTable.deleteRow ((IBlueRow)myDict[groupRepString]!);
+         myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
          myDict.Remove (groupRepString);
       }
 
@@ -51,8 +48,11 @@ namespace BCG
          myDict.Clear ();
       }
 
-      public IBlueRow? getRow(string groupRepString) =>
-         myDict[groupRepString] as IBlueRow;
+      public IBlueRow getRow (string groupRepString)  
+      {
+         return myDict[groupRepString] == null ? null : 
+            myDict[groupRepString] as IBlueRow;
+      }
 
       #region IBlueRowCollection members
       public IBlueRow this[int id]  {get{return myTable[id];}}

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -1,9 +1,7 @@
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
+using BCG.Utilities;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
 namespace BCG {
@@ -28,12 +26,11 @@ namespace BCG {
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      RowDictionaryTable? myBookKeepingTable;
+      private RowDictionaryTable myBookKeepingTable;
       private IGroupByEvaluator myGroupByEvaluator;
-      // input schema, as output schema can differ (IGroupByEvaluator's
-      IBlueTableSchema? mySchema;
+      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
       //can map a schema to another
-      IBlueTableSchema? myOutputSchema;
+      private IBlueTableSchema myOutputSchema;
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
@@ -49,14 +46,14 @@ namespace BCG {
       }
 
       protected override void myHandleFirstCall() {
-         requireLastArrivedPacket();
          mySchema = myLastArrivedPacket.schema;
          myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
          myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
             myGroupByEvaluator.groupByColumns);
          GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
          foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            var newRow = myEvaluate(new DataPacket(rowGroup, rowGroup, new BlueTable(mySchema)));
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
             if (newRow != null)
                temp.addRow(newRow, myGetGroupString(rowGroup));
          }
@@ -82,14 +79,15 @@ namespace BCG {
          ArrayList handledAddGroups = new ArrayList();
          string groupRepString;
 
-         requireBookKeeping();
          foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
             // BlueEventLog.logAssert (delGroup.Count > 0);
             groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) ?? new BlueTable(mySchema);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
             handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) ?? new BlueTable(mySchema);
-            var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
             myFireSwitch |= myMergeRow(newRow, groupRepString);
          }
 
@@ -101,16 +99,16 @@ namespace BCG {
                //the loop above
 
                groupRepString = myGetGroupString(addGroup);
-               var aggGroup = ExceptionAssert.NotNull(myBookKeepingTable.getGroup(groupRepString));
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, new BlueTable(aggGroup.schema)));
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
                myFireSwitch |= myMergeRow(newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow(IBlueRow? newRow, string groupRepString) {
-         requireLastFiredPacket();
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
          IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
@@ -118,7 +116,6 @@ namespace BCG {
          else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
             return false;
          else if (existingRow == null) {
-            ArgumentNullException.ThrowIfNull(newRow);
             //row added for the first time
             ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
             ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
@@ -141,9 +138,6 @@ namespace BCG {
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
       private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
-         ExceptionAssert.NotNull(myBookKeepingTable);
-         requireLastFiredPacket();
-         requireLastArrivedPacket();
          delGroups =
             new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
@@ -157,19 +151,16 @@ namespace BCG {
       }
 
       protected override void myOnTableClear() {
-         if (ExceptionAssert.NotNull(myBookKeepingTable).Count == 0) {
-            // ie there arent any rows, so no need to optimize the 'cleansing' process
-            return;  
-         }
-         requireLastFiredPacket();
+         if (myBookKeepingTable.Count == 0)
+            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
          myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         ((GroupByOutputTable)myLastFiredPacket.newRows).clear();
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
          myBookKeepingTable.clear();
          myGroupByEvaluator.clear();
       }
 
-      private IBlueRow? myEvaluate(DataPacket dp) {
+      private IBlueRow myEvaluate(DataPacket dp) {
          IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
          return evalRow == null ? null :
             myIdGuard ?
@@ -201,7 +192,7 @@ namespace BCG {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow? getRepRow(IDataPacket dp) {
+      public static IBlueRow getRepRow(IDataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
@@ -220,13 +211,13 @@ namespace BCG {
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object? incrementalAdd(object lastValue, object newValue, bool isAdd) {
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
          if (lastValue == null || lastValue.Equals(DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
          // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
          // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
-         object? retVal = null;
+         Object retVal = null;
          if (newValue.GetType().Equals(typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
          else if (newValue.GetType().Equals(typeof(Double)))
@@ -238,9 +229,6 @@ namespace BCG {
          return retVal;
       }
 
-      [MemberNotNull(nameof(myBookKeepingTable))]
-      void requireBookKeeping() =>
-         ExceptionAssert.NotNull(myBookKeepingTable);
    }
 
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,3 +1,8 @@
+using System;
+using System.Collections;
+
+using BCG.Utilities;
+
 namespace BCG {
     /// <summary>
     /// Computes the max for a given column
@@ -12,10 +17,10 @@ namespace BCG {
         public override IBlueRow? evaluate(IDataPacket dp) {
             myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            var currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
             if (maxRow != null) {
-                bts.Log.notnull(currMaxRow);
+                bts.Log.assert(currMaxRow != null);
                 bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
                 if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
                     maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/MethodDrivenStream.cs

@@ -1,9 +1,10 @@
+using System;
+using System.Collections;
+using BCG.Utilities;
+using System.IO;
+
 namespace BCG
 {
-   using Spark.Core.Diagnostics;
-
-   using System.Diagnostics.CodeAnalysis;
-
    /// <summary>
    /// Some basic event handling functionality factorred out of any Stream whose behaviour
    /// is driven by some interface passed on by the user
@@ -11,8 +12,8 @@ namespace BCG
    public abstract class MethodDrivenStream : ViewStream
    {
       protected ViewStream myStream;
-      protected DataPacket? myLastArrivedPacket;
-      protected DataPacket? myLastFiredPacket;
+      protected DataPacket myLastArrivedPacket;
+      protected DataPacket myLastFiredPacket;
       protected bool myFireSwitch = true;
 
       public MethodDrivenStream(ViewStream vs)  {myStream = vs;}
@@ -58,7 +59,6 @@ namespace BCG
       /// To handle the first call, the LastFiredPacket is supposed to be initialized and filled
       /// with data.
       /// </summary>
-      [MemberNotNull(nameof(myLastFiredPacket))]
       protected abstract void myHandleFirstCall ();
 
       /// <summary>
@@ -89,13 +89,5 @@ namespace BCG
       {
          myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
       }
-
-      [MemberNotNull(nameof(myLastArrivedPacket))]
-      protected void requireLastArrivedPacket() =>
-         ExceptionAssert.NotNull(myLastArrivedPacket);
-
-      [MemberNotNull(nameof(myLastFiredPacket))]
-      protected void requireLastFiredPacket() =>
-         ExceptionAssert.NotNull(myLastFiredPacket);
    }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -9,7 +9,7 @@ namespace BCG {
         public override IBlueRow? evaluate(IDataPacket dp) {
             myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
-            var currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
             IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
             if (minRow != null) {
                 // BlueEventLog.logAssert (currMinRow != null);

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -45,13 +45,12 @@ namespace BCG {
         }
 
         public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
-            ViewStream? rs = null;
+            ViewStream rs = null;
             groupby_columns = groupby_columns ?? new string[] { };
             if (groupby_columns.Length == 0) {
                 // rs = new AddColumnsStream(stream, "fake_col=0");
                 // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
                 // rs = new ProjectStream(rs, "!fake_col");
-                throw new ArgumentException("No groupby columns specified", nameof(groupby_columns));
             } else {
                 rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
             }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -1,7 +1,5 @@
 using mbts;
 
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections.Generic;
 
@@ -37,7 +35,7 @@ namespace BCG {
          List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
          foreach (Triplet<ViewStream, string, string> r in join_specs) {
             string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = ExceptionAssert.NotNull(r.third).Split(' ');
+            string[] visible_cols = r.third.Split(' ');
             List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
             foreach (string s in visible_cols) {
                string[] toks = s.Split(':');
@@ -49,7 +47,7 @@ namespace BCG {
                   mapped_col_names.Add(toks[1]);
                }
             }
-            rv.Add(new(ExceptionAssert.NotNull(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
          }
          return rv.ToArray();
       }

src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs

@@ -290,7 +290,6 @@ namespace BCG
                }
             }
          }
-         ExceptionAssert.NotNull(myState);
       }
 
       private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)

src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinForeignStream.cs

@@ -1,5 +1,3 @@
-using Spark.Core.Diagnostics;
-
 using System;
 
 namespace BCG
@@ -20,21 +18,16 @@ namespace BCG
 
       public override void onTableClear (bool isLeft)
       {
-         requireSchema();
-         requireLastFiredPacket();
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);
-         if (!isLeft) {
-            ExceptionAssert.NotNull(myLeftDataPacket);
+         if (!isLeft)
             foreach (IBlueRow row in myLeftDataPacket.newRows)
                ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
-         }
       }
 
       protected override void myInitFiredPacket ()
       {
-         requireSchema();
          myLastFiredPacket = new DataPacket ();
          myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
          myLastFiredPacket.addedRows = new BlueTable (mySchema);

src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs

@@ -15,7 +15,7 @@ namespace BCG
          : this (r, null, null)
       {}
 
-      public SimpleNode(IBlueRow? r, SimpleNode? nx, SimpleNode? pv) {
+      public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
          row = r;
          next = nx;
          prev = pv;

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -1,7 +1,5 @@
 using bts;
 
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections.Generic;
 
@@ -62,7 +60,7 @@ namespace BCG {
             if (columnAndType.Length == 1)
                ci[n] = new ColumnInfo(c, n, typeof(double));
             else
-               ci[n] = new(columnAndType[0], n, ExceptionAssert.NotNull(Type.GetType("System." + columnAndType[1], true)));
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
             n++;
          }
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -44,7 +44,7 @@ namespace bts {
          BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
          :
          new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
-         BlueTable? out_table = null;
+         BlueTable out_table = null;
          foreach (var kv in groups) {
             List<IBlueRow> group_rows = kv.Value;
             group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
@@ -73,4 +73,4 @@ namespace bts {
    }
 
 
-}
+}

src/DataStreamSuite/DataStream/Primitives/Map/ISuperStreamMapper.cs

@@ -23,6 +23,6 @@ namespace BCG
       /// fires an event when the mapper 
       /// changes itself
       /// </summary>
-      event MapperChangeDelegate? onMapperChange;
+      event MapperChangeDelegate onMapperChange;
    }
 }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -75,7 +75,6 @@ namespace BCG {
 
       protected override void myHandleFirstCall() {
          ArgumentNullException.ThrowIfNull(myMapper);
-         requireLastArrivedPacket();
          IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
          myLastFiredPacket = new DataPacket();
          myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -86,12 +85,10 @@ namespace BCG {
       }
 
       protected override void myHandleIncrementalAdd() {
-         requireLastArrivedPacket();
          if (myLastArrivedPacket.addedRows.Count == 0) {
             Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
             return;
          }
-         requireLastFiredPacket();
          ArgumentNullException.ThrowIfNull(myMapper);
          var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
          var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
@@ -104,12 +101,10 @@ namespace BCG {
       }
 
       protected override void myHandleIncrementalDelete() {
-         requireLastArrivedPacket();
          if (myLastArrivedPacket.deletedRows.Count == 0) {
             Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
             return;
          }
-         requireLastFiredPacket();
          var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
          var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
          foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
@@ -132,7 +127,6 @@ namespace BCG {
       }
 
       protected override void myOnTableClear() {
-         requireLastFiredPacket();
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
          myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -47,14 +47,14 @@ namespace BCG {
             _map_rows = map_rows;
          }
 
-         event MapperChangeDelegate? ISuperStreamMapper.onMapperChange { add { } remove { } }
-
          public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
 
          public IBlueRow[] map(IBlueRow r) {
             IEnumerable<IBlueRow> rows = _map_rows(r);
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
+
+         public event MapperChangeDelegate? onMapperChange;
       }
 
       //maps the id of the underlying stream to the

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -608,8 +608,8 @@ namespace BCG {
             // 5. deserialize the state into a stream op
             str_io_msg_p = deser.deserialize(content_p);
             // 6. check to make sure the state's rows are good
-            passes &= str_io_msg_p.op == StreamIOMessage.StreamOp.CREATE
-               && BlueDataUtility.compute_equality(cr_rows, ExceptionAssert.NotNull(str_io_msg_p.create_message).rows);
+            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
+                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
             return new TestResult {
                 test_result = passes,
                 test_name = "run_pubsub_server_stream_tag_handler_test"
@@ -692,7 +692,7 @@ namespace BCG {
             var content = st_data.get_current_content();
             var msg_p = deser.deserialize(content);
             Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
-            return ExceptionAssert.NotNull(msg_p.create_message).rows;
+            return msg_p.create_message.rows;
 
         }
 

src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs

@@ -1,7 +1,5 @@
 using Bts.ColumnSortExpressions;
 
-using Spark.Core.Diagnostics;
-
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -41,7 +39,7 @@ public class SortRow : IComparable, IBlueRow {
          var otherVal = otherRow[col.Name];
          int rv = myIsNull(this[col.Name]) ?
             (myIsNull(otherVal) ? 0 : -1) :
-            (myIsNull(otherVal) ? 1 : ExceptionAssert.Argument.Is<IComparable>(this[col.Name]).CompareTo(otherVal));
+            (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
          rv *= col.IsAscending ? 1 : -1;
          return rv;
       }

src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs

@@ -1,6 +1,4 @@
-using Spark.Core.Diagnostics;
-
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 
@@ -62,13 +60,13 @@ namespace BCG
             List<string> columnInfoStrParts = new List<string>(columnInfoStr.Split(new char[] { ':' }, StringSplitOptions.None));
             if (columnInfoStrParts.Count != 2)
             {
-               throw new Exception($"Unable to decode schema string '{schemaStr}'");
+               throw (new Exception(String.Format("Unable to decode schema string '{0}'", schemaStr)));
             }
 
-            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, ExceptionAssert.NotNull(Type.GetType(columnInfoStrParts[1]))));
+            columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, Type.GetType(columnInfoStrParts[1])));
          }
 
-         return new BlueTableSchema(columnInfos.ToArray(), string.Empty);
+         return (new BlueTableSchema(columnInfos.ToArray(), ""));
       }
 
       private IBlueRow myGetRow(IBlueTableSchema schema, string rowStr)
@@ -77,7 +75,7 @@ namespace BCG
 
          if (schema.columnInfo.Length != valueStrs.Count)
          {
-            throw new Exception($"Unable to decode row string '{rowStr}'");
+            throw (new Exception(String.Format("Unable to decode row string '{0}'", rowStr)));
          }
 
          List<object> values = new List<object>();
@@ -86,7 +84,7 @@ namespace BCG
             values.Add(Convert.ChangeType(valueStrs[i], schema.columnInfo[i].colType));
          }
 
-         return new BlueRow(schema, values.ToArray());
+         return (new BlueRow(schema, values.ToArray()));
       }
       #endregion
 

src/DataStreamSuite/DataStream/Primitives/ThreadSafe/VSDBMarshallerStream.cs

@@ -1,7 +1,6 @@
 using BCG.Utilities.General;
 
 using System;
-using System.Collections.Generic;
 using System.Threading;
 
 namespace BCG
@@ -98,7 +97,7 @@ namespace BCG
          // BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread, 
          //    "onTargetDoWork should only be called from target thread");
          var updates = _queued_updates.dequeueAll();
-         foreach (var u in updates.ExceptNull()) {
+         foreach (var u in updates) {
             if (u.subscription == _current_target_subscription)
                myFireEvent(u.dp);
          }

src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs

@@ -1,9 +1,7 @@
-using BCG.Utilities.General;
-
-using Spark.Core.Diagnostics;
-
 using System;
 
+using BCG.Utilities.General;
+
 namespace BCG {
    public abstract class TimeUnitStream : ViewStream, IWorker {
       protected TimeSpan myOffset;
@@ -48,8 +46,8 @@ namespace BCG {
             myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
 
 
-         var latestRow = myGetLatestRow();
-         if (latestRow is null || !ExceptionAssert.Argument.Is<DateTime>(latestRow["time"]).Equals(dt)) {
+         if (myGetLatestRow() == null ||
+            !myGetLatestRow()["time"].Equals(dt)) {
             IBlueRow ar = new BlueRow(myCollection.schema,
                myId++, dt);
             IBlueRow dr = myCollection.addRowWithReturn(ar);
@@ -73,10 +71,11 @@ namespace BCG {
 
       #region Private Helper methods
 
-      IBlueRow? myGetLatestRow() =>
-         myCollection.Count == 0
-            ? null
-            : BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
+      private IBlueRow myGetLatestRow() {
+         return myCollection.Count == 0 ?
+            null :
+            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
+      }
 
       private IBlueRowCollection myConvertToCollection(IBlueRow r) {
          BlueTable retVal = new BlueTable(myCollection.schema);

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,42 +1,43 @@
-namespace BCG;
+using System;
 
-public class TopStream : MethodDrivenStream {
-   int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-   public TopStream(ViewStream stream, int topCount)
-      : base(stream) =>
-      myTopCount = topCount;
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-   public TopStream(ViewStream stream, int topCount, string sortExpression)
-      : this(sortExpression is null ? stream : new SortStream(stream, sortExpression), topCount) { }
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-   protected override void myOnTableClear() {
-      requireLastFiredPacket();
-      myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-      myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
-   }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-   protected override void myHandleFirstCall() {
-      requireLastArrivedPacket();
-      myLastFiredPacket = new DataPacket();
-      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-      myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-      myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
-   }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-   protected override void myHandleIncrementalAdd() { }
+        protected override void myHandleIncrementalAdd() { }
 
-   protected override void myHandleIncrementalDelete() {
-      requireLastArrivedPacket();
-      requireLastFiredPacket();
-      IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-      myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
-      BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
-         oldRows,
-         myLastFiredPacket.schema.columnNames,
-         out myLastFiredPacket.deletedRows,
-         out myLastFiredPacket.addedRows);
-      myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-   }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
 }

src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs

@@ -1,10 +1,12 @@
+
+using System;
 using System.Collections;
 
 namespace BCG {
 
    public class AlphabetizeColumnsStream : ViewStream {
       private ViewStream mySrc;
-      IBlueTableSchema? mySchema = null;
+      private IBlueTableSchema mySchema = null;
 
       public AlphabetizeColumnsStream(ViewStream src) {
          mySrc = src;

src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs

@@ -1,12 +1,12 @@
-using System.Linq;
+using System;
 
 namespace BCG
 {
 	public class AppendStreamModified : ViewStream
 	{
       private AppendCollection myColl = new AppendCollection ();
-      ViewStream? myLeftStream, myRightstream;
-
+      private ViewStream myLeftStream, myRightstream;
+	
       public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
 		{
          myLeftStream = leftStream;
@@ -15,24 +15,11 @@ namespace BCG
 
       public AppendStreamModified (ViewStream[] vss)
       {
-         switch (vss.Length) {
-            case 0:
-               break;
-            case 1:
-               mySetRealSource (vss[0]);
-               break;
-            default:
-               mySetRealSource(
-                  vss[1..].Aggregate(
-                     new AppendStreamModified(new[] { vss[0] }),
-                     (asm, vs) => new AppendStreamModified(asm, vs)));
-               break;
-         }
          if (vss.Length == 1)
             mySetRealSource (vss[0]);
          else
          {
-            AppendStreamModified? rv = null;
+            AppendStreamModified rv = null;
             foreach (ViewStream s in vss)
                rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s); 
             mySetRealSource (rv);

src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs

@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
 
+using global::Spark.Core.Disposables;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
 using System.Linq;

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -8,6 +8,7 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
+using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;
 

src/DataStreamSuite/DataStreamSuite.sln

@@ -43,14 +43,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "..\cs\Bts.Collections\Bts.Collections.csproj", "{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "..\cs\Bts.Core\Bts.Core.csproj", "{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "..\cs\Bts.Disposables\Bts.Disposables.csproj", "{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "..\cs\Bts.Numerics\Bts.Numerics.csproj", "{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -301,54 +293,6 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.Build.0 = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.Build.0 = Debug|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.ActiveCfg = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.Build.0 = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.ActiveCfg = Release|Any CPU
-		{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.Build.0 = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.Build.0 = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.Build.0 = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.Build.0 = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.Build.0 = Debug|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.ActiveCfg = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.Build.0 = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.ActiveCfg = Release|Any CPU
-		{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.Build.0 = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.Build.0 = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.Build.0 = Debug|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.ActiveCfg = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.Build.0 = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.ActiveCfg = Release|Any CPU
-		{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStreamTests/TestDriverStream.cs

@@ -5,7 +5,7 @@ using BCG.Utilities.General;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
+using global::Spark.Core.Collections;
 
 public class TestDriverStream : ViewStream, IWorker {
    readonly IBlueTableSchema schema;

src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs

@@ -22,6 +22,7 @@ namespace BCG {
    /// with PubSub, as some clients are still using ManagedSubcribers)
    /// </summary>
    public class ManagedSubscriber : IManagedSubscriber, IWorker, IDisposable {
+      private static ManagedSubscriber? _self;
       private ISubscriberCore _subcore;
       // private IDictionary myTypedDispatchTable = new Hashtable(),
       //    myRawDispatchTable = new Hashtable();
@@ -33,6 +34,7 @@ namespace BCG {
       ManagedSubscriber(ISubscriberCore subsCore) { _subcore = subsCore; }
 
       public void doWork() {
+         var msg_map = new Dictionary<string, List<bts.PubSubServerMessage>>();
          var updates = _subcore.get_updates(TimeSpan.Zero);
          if (updates.Count() == 0)
             return;

src/DataStreamSuite/PubSub/ManagedTCP/RemoteMessage.cs

@@ -45,7 +45,11 @@ namespace BCG
          get { return myConnInfo; }
       }
 
-      public override string ToString() =>
-         $"type: {myMsgType} conn: {myConnInfo} msg: {myMsg?.Length ?? 0}";
+      override public string ToString()
+      {
+         Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
+         return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
+      }
+
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs

@@ -46,7 +46,7 @@ namespace bts {
             System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
             _accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { });
             if (ex is OutOfMemoryException)
-               throw;
+               throw ex;
             return;
          }
          new_sock.LingerState = new LingerOption(false, 0);

src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs

@@ -1,8 +1,9 @@
-using bts;
-
 using System;
 using System.IO;
-using System.Text;
+using System.Collections;
+using bts;
+using System.Threading;
+using System.Reflection;
 
 namespace BCG {
    /// <summary>
@@ -68,8 +69,7 @@ namespace BCG {
 
       public static byte[] getBytesFromStream(params Stream[] streams) {
          long totalSize = 0;
-         using var ms = new MemoryStream();
-         using var bw = new BinaryWriter(ms, Encoding.UTF8, true);
+         BinaryWriter bw = new BinaryWriter(new MemoryStream());
          foreach (Stream s in streams)
             totalSize += s.Length - s.Position;
          bw.Write(Convert.ToInt32(totalSize));
@@ -78,8 +78,7 @@ namespace BCG {
             int lengthToReadForThisStream = Convert.ToInt32(s.Length - s.Position);
             bw.Write(br.ReadBytes(lengthToReadForThisStream));
          }
-         bw.Flush();
-         return ms.ToArray();
+         return (bw.BaseStream as MemoryStream).ToArray();
       }
 
       public static byte[] read_bytes(Stream str) {

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -92,7 +92,7 @@ namespace BCG {
 
       public void publish(PubSubClientMessage msg) {
          Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
-         Log.assert(!string.IsNullOrEmpty(msg.pub_msg?.tag_type));
+         Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
          bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
          _main_queue.enqueue(msg);
       }

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -143,7 +143,7 @@ namespace BCG {
       public void doWork() {
          if (_connection_cb != null) {
             var conns = _conn_callback_queue.dequeueAll();
-            foreach (var ci in conns.ExceptNull()) {
+            foreach (var ci in conns) {
                _connection_cb(ci.client_url, ci.status);
             }
          }
@@ -175,7 +175,7 @@ namespace BCG {
             case RemoteMessageType.Data:
                // if callback is registered fire it right away, else enqueue it
                if (_data_cb != null)
-                  _data_cb(sender_rci.ToString(), message.message ?? Array.Empty<byte>());
+                  _data_cb(sender_rci.ToString(), message.message);
                else
                   _received_data_queue.enqueue(new Pair<string, byte[]>(sender_rci.ToString(), message.message));
                break;
@@ -258,7 +258,7 @@ namespace BCG {
       // (which gets called in another thread)
       private void _handle_new_connections() {
          var new_conns = _new_connections.dequeueAll();
-         foreach (var conn in new_conns.ExceptNull()) {
+         foreach (ITCPConnection conn in new_conns) {
             var peer_rci = conn.get_peer_address();
             bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections conn: {conn}");
             ///to stop it from being garbabge collected immediatly

src/DataStreamSuite/PubSub/Server/IPubSubState.cs

@@ -16,8 +16,8 @@ public interface IPubSubState {
 
    ICollection<string> all_tags { get; }
 
-   byte[]? get_content(string tag);
-   string? getTagType(string tag);
+   byte[] get_content(string tag);
+   string getTagType(string tag);
    void get_all_connections(
       string tag,
       out Dictionary<string, string> subscribers,

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -188,11 +188,11 @@ namespace BCG {
          _state.handle_sub(sub_msg);   // state keeps the list of all subscribers
          ///to all these new subscriber s, send the existing data
          foreach (string tag in sub_msg.tags.empty_if_null()) {
-            byte[]? data = _state.get_content(tag);
+            byte[] data = _state.get_content(tag);
             //ie data has been published once
             if (data != null) {
                bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
-               string? tag_type_ = _state.getTagType(tag);
+               string tag_type_ = _state.getTagType(tag);
                var msg_to_send = new PubSubServerMessage() {
                   tag = tag,
                   tag_type = tag_type_,

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -148,7 +148,7 @@ namespace BCG {
             return new string[] { };
       }
 
-      public byte[]? get_content(string tag) {
+      public byte[] get_content(string tag) {
          if (!_tag_to_data_map.ContainsKey(tag))
             return null;
          var rv = _tag_to_data_map[tag].get_current_content();
@@ -157,7 +157,7 @@ namespace BCG {
          return rv;
       }
 
-      public string? getTagType(string tag) {
+      public string getTagType(string tag) {
          if (!_tag_to_data_map.ContainsKey(tag))
             return null;
          else

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -20,7 +20,7 @@ namespace bts {
    public class SingleTagData {
       private readonly IPubSubServerTagHandler? _tag_handler;
 
-      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler? tag_handler) {
+      public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
          bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
          this.tag = tag;
          this.tag_type = tag_type;

src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs

@@ -46,6 +46,7 @@ namespace BCG {
       }
 
       public void onConnection(ITCPConnection connection) {
+         string ip; int port;
          var peer_rci = connection.get_peer_address();
          bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop");
          if (_listener != null) {

src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookSchemaBuilder.cs

@@ -5,10 +5,9 @@ using BCG;
 using bts.api;
 using bts.md;
 
-using Bts.DataStream;
 using Bts.DataStreamCore;
 
-using Spark.Api.MarketData;
+using global::Spark.Api.MarketData;
 
 public class BookSchemaBuilder {
    public int NumLevels { get; set; } = 5;

src/DataStreamSuite/SharedStreams/BlueOrb/WellKnownColumns.cs

@@ -2,7 +2,7 @@ namespace Bts.SharedStreams;
 
 using bts.api;
 
-using Spark.Api.Orders;
+using global::Spark.Api.Orders;
 
 public static class WellKnownColumns {
    public static (string Name, Type Type) AskNotional => (WellKnownColumnNames.AskNotional, typeof(double));

src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs

@@ -3,7 +3,6 @@ using bts.config;
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 
 namespace BCG {
 
@@ -22,9 +21,7 @@ namespace BCG {
     }
     public class RenkoCalculator {
         public readonly double brick_size;
-        public RenkoBrick? current_brick;
-
-        [MemberNotNull(nameof(current_brick))]
+        public RenkoBrick current_brick = null;
         private bool _add_price(double price) {
             if (current_brick == null) {
                 current_brick = new RenkoBrick {
@@ -312,4 +309,4 @@ namespace BCG {
             return wma_values;
         }
     }
-}
+}

src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs

@@ -10,7 +10,7 @@
    [DataStreamUserRespresentation("nl")]
    public class BlueOrbNLStream : ViewStream {
       private readonly string _nl_prompt;
-      private ViewStream? _nl_to_dsl_stream, _dsl_stream;
+      private ViewStream _nl_to_dsl_stream, _dsl_stream;
       private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
       public BlueOrbNLStream(string nl_prompt) {
          _nl_prompt = nl_prompt;

src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs

@@ -85,8 +85,7 @@ namespace bts.BlueOrb {
                 .GetProperty("choices")[0]
                 .GetProperty("message")
                 .GetProperty("content")
-                .GetString()
-                ?? string.Empty;
+                .GetString();
 
             //return gpt_response;
             DateTime t1 = DateTime.Now;

src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs

@@ -4,8 +4,6 @@ using BCG.Utilities.General;
 using Bts.PubSub;
 using Bts.SharedStreams.Tables;
 
-using Spark.Core.Diagnostics;
-
 namespace bts {
    // TODO: Use dependency injection to instantiate this stream, and others like it that depend on
    // ISubscriberCore(Factory). Until then, hide it from DSQA.
@@ -42,7 +40,7 @@ namespace bts {
          foreach (var u in updates) {
             if (u.tag != "System_AllStatistics")
                continue;
-            var tag_infos = TagStatsInfo.deserialize(ExceptionAssert.NotNull(u.data));
+            var tag_infos = TagStatsInfo.deserialize(u.data);
             foreach (var ti in tag_infos) {
                _tag_map[ti.tag] = ti;
             }

src/Directory.Build.targets

@@ -187,32 +187,24 @@
       Name="Test"
       DependsOnTargets="Build"></Target>
 
-  <ItemGroup Condition="Exists('@(NativeWindowsAsset)')">
-    <Content Include="@(NativeWindowsAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>%(FileName)%(Extension)</TargetPath>
-      <Pack>false</Pack>
-    </Content>
-    <Content Include="@(NativeWindowsAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>runtimes/win-x64/native/%(FileName)%(Extension)</TargetPath>
-      <Pack>true</Pack>
-      <PackagePath>runtimes/win-x64/native</PackagePath>
-    </Content>
-  </ItemGroup>
+  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
+    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
+    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
+  </Target>
 
-  <ItemGroup Condition="Exists('@(NativeLinuxAsset)')">
-    <Content Include="@(NativeLinuxAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>%(FileName)%(Extension)</TargetPath>
-      <Pack>false</Pack>
-    </Content>
-    <Content Include="@(NativeLinuxAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>runtimes/linux-x64/native/%(FileName)%(Extension)</TargetPath>
-      <Pack>true</Pack>
-      <PackagePath>runtimes/linux-x64/native</PackagePath>
-    </Content>
+  <ItemGroup>
+    <Content Include="@(NativeWindowsAsset)"
+             Condition="Exists('@(NativeWindowsAsset)')"
+             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/win-x64/native/" />
+    <Content Include="@(NativeLinuxAsset)"
+             Condition="Exists('@(NativeLinuxAsset)')"
+             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/linux-x64/native/" />
   </ItemGroup>
 
 </Project>

src/app_sparkd/Dockerfile

@@ -2,14 +2,14 @@ ARG IMAGE="sparkd"
 ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
 ARG SOURCE_DIR="install/sparkd-release-gcc"
 ARG PYTHON_REQUIREMENTS="requirements.txt"
-ARG ENTRYPOINT_SCRIPT="src/app_sparkd/sparkd-entrypoint.sh"
+ARG SPARKD_ENTRYPOINT="src/app_sparkd/sparkd-entrypoint.sh"
 
 # sparkd_base
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_base
 ARG CONFIG_DIR
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS
-ARG ENTRYPOINT_SCRIPT
+ARG SPARKD_ENTRYPOINT
 
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -23,7 +23,7 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
 COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/.local/share/bts/sparkd/conf"
-COPY --chown=bts:bts "${ENTRYPOINT_SCRIPT}" "/home/bts/bin/sparkd-entrypoint.sh"
+COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"
 
 # sparkd
@@ -38,7 +38,7 @@ COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/.local/share/bts/sparkd/da
 FROM bluetradingsystems/linux_dotnet8 AS sparkd_symbols
 ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS="requirements.txt"
-ARG ENTRYPOINT_SCRIPT
+ARG SPARKD_ENTRYPOINT
 
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
@@ -51,9 +51,8 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
 
 USER "bts"
 COPY --chown=bts:bts "${SOURCE_DIR}" "."
-COPY --chown=bts:bts "${ENTRYPOINT_SCRIPT}" "/home/bts/bin/sparkd-entrypoint.sh"
+COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
 WORKDIR "/home/bts/bin"
 
 # selected image
 FROM ${IMAGE}
-ENTRYPOINT "./sparkd-entrypoint.sh"

src/app_sparkd/configurations/cme_replay/.env

@@ -1,4 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_replay
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_replay/requirements.txt
-SPARKD__ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
-TZ=America/Chicago

src/app_sparkd/configurations/cme_replay/spark.conf

@@ -1,38 +0,0 @@
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-MarketDataConfig {
-  value {
-   # glbx{
-   #   use_databento: true
-   #  databento_settings {
-   #     read_from_file: true
-   #    key: "db-DgxpuTXneNgVvd7sJCAEKYDhyCApW"
-   #     bbo_file_name: "glbx-mdp3-20251103.mbp-1.dbn.zst"
-   #     contracts_file_name: "glbx-mdp3-20251103.definition.dbn.zst"
-   #   }
-   # }
-   # recording{
-   #    venue: VENUE_GLBX
-   #     num_of_seconds_to_run_file_writer: 240
-   #     num_of_seconds_to_run_db_writer : 0
-   #     n_book_levels: 10 
-   #     batch_size: 100 #for the db writer
-   # }
-   replay {
-      venue: VENUE_GLBX
-    }
-  }
-}

src/app_sparkd/configurations/cme_replay/symbols.conf

@@ -1,48 +0,0 @@
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: "ES"
-        # option: "E1A"
-        # option: "E1B"
-        # option: "E1C"
-        # option: "E1D"
-        # option: "E2A"
-        # option: "E2B"
-        # option: "E2C"
-        # option: "E2D"
-        # option: "E3A"
-        # option: "E3B"
-        # option: "E3C"
-        # option: "E3D"
-        # option: "E4A"
-        # option: "E4B"
-        # option: "E4C"
-        # option: "E4D"
-        # option: "E5A"
-        # option: "E5B"
-        # option: "E5C"
-        # option: "E5D"
-        # option: "EW1"
-        # option: "EW2"
-        # option: "EW3"
-        # option: "EW4"
-        # option: "ESH9"
-        # option: "ESZ6"
-        # option: "ESH0"
-        # option: "ESM6"
-        # option: "ESZ7"
-        # option: "ESH6"
-        # option: "ESM7"
-        # option: "ESZ8"
-        # option: "ESH7"
-        # option: "ESH8"
-        # option: "ESU6"
-        # option: "ESZ5"
-        # option: "ESZ9"
-        # option: "E1D"
-        # option: "ESZ5"
-        # option: "ES"
-    }
-  }
-}

src/app_sparkd/configurations/ophir_cert_live_wb/.env

@@ -0,0 +1,10 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
+SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g
+SPARKD__CONFIG_PORT=9000
+SPARKD__QUICKBUS_PORT=9001
+SPARKD__SIM_FIX_PORT=9002

src/app_sparkd/configurations/ophir_cert_live_wb/spark.conf

@@ -0,0 +1,140 @@
+DeploymentType {
+  value {
+    type: "dev"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'ens65f1'
+      network_interface_b: 'ens65f0'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "maw_004_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "69.50.112.177"
+      socket_connect_port: 44040
+      sender_compid: "MAW004"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 2
+      max_connection_attempts: 100
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}
+Ports {
+   value {
+      config_port: 9000
+      qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
+      replicator_port: 9080
+   }
+}

src/app_sparkd/configurations/ophir_demo/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_demo/symbols.conf

@@ -1,8 +0,0 @@
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GLBX"
-      future: "ES"
-    }
-  }
-}

src/app_sparkd/configurations/ophir_prod_live_wb/.env

@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
+SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_prod_live_wb/spark.conf

@@ -0,0 +1,152 @@
+DeploymentType {
+  value {
+    type: "prod"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'enp193s0f0np0'
+      network_interface_b: 'enp193s0f1np1'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "u6p_7qb_54"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.97"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "54"
+      cme_market_segment_id: 54
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_order_msgs_per_window: 300
+      max_connection_attempts: 2
+      use_ilink3: true
+    }
+   connection {
+      name: "u6p_7qb_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.1"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_connection_attempts: 2
+      max_order_msgs_per_window: 300
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}

src/app_sparkd/configurations/ophir_prod_live_wb/symbols.conf

@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+      symbol: "ESH9"
+      symbol: "ESZ6"
+      symbol: "ESH0"
+      symbol: "ESM6"
+      symbol: "ESZ7"
+      symbol: "ESH6"
+      symbol: "ESM7"
+      symbol: "ESZ8"
+      symbol: "ESH7"
+      symbol: "ESH8"
+      symbol: "ESU6"
+      symbol: "ESZ5"
+      symbol: "ESZ9"
+      symbol: "E1D"
+      symbol: "ESZ5"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}

src/app_sparkd/configurations/ophir_sim/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
-SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_sim/spark.conf

@@ -1,104 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-   value {
-      network_interface_a: 'ens3f0np0'
-      network_interface_b: 'ens3f1np1'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-  }
-}
-
-# JetFIX {
-#    value {
-#     #connection: @JetFIXSimConnection.sim
-#     connection {
-#     name: "mlx_004_54"
-#     venue: VENUE_GLBX
-#     socket_connect_host_ip: "69.50.112.191"
-#     socket_connect_port: 42876
-#     sender_compid: "MLX004"
-#     target_compid: "CME"
-#     target_subid: "54"
-#     cme_market_segment_id: 54
-#     start_time: "00:10:00"
-#     end_time: "23:55:00"
-#     start_day: "sunday"
-#     end_day: "friday"
-#     seconds_between_hbs: 30
-#     sender_subid: "BTS"
-#     use_ilink3: true
-#   #   raw_data: "S4PMA"
-#   #   auto_fix_seq_num: true
-#   }
-#    }
-# }
-
-

src/app_sparkd/configurations/ophir_sim_replay/.env

@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay/requirements.txt
+TZ=America/Chicago
+SPARKD__MD_RECORDING_DIR=/home/wedbushprod/8000/md_recording

src/app_sparkd/configurations/ophir_sim_replay/spark.conf

@@ -24,38 +24,9 @@ PublisherOptions {
   }
 }
 
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Warn"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "OrderMan"
-        level: "Diag"
-      }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
-    }
-  }
-}
-
 MarketDataConfig {
   value {
-    replay {
+   replay {
       venue: VENUE_GLBX
     }
   }

src/app_sparkd/configurations/ophir_sim_replay/symbols.conf

@@ -0,0 +1,46 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "ES"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}

src/app_sparkd/configurations/ophir_sim_replay_wb/.env

@@ -0,0 +1,9 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay_wb
+SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
+SPARKD__MD_RECORDING_DIR=/home/wedbushprod/md_recording
+SPARKD__CONFIG_PORT=9000
+SPARKD__QUICKBUS_PORT=9001
+SPARKD__SIM_FIX_PORT=9002

src/app_sparkd/configurations/ophir_sim_replay_wb/requirements.txt

@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1

src/app_sparkd/configurations/ophir_sim_replay_wb/spark.conf

@@ -0,0 +1,46 @@
+JetFIX {
+  value {
+    connection {
+      name: "glbx_1"
+      venue: VENUE_GLBX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+Ports {
+   value {
+      config_port: 9000
+      qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
+   }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+     replay {
+      venue: VENUE_GLBX
+    }
+  }
+}
+

src/app_sparkd/configurations/ophir_sim_replay_wb/symbols.conf

@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+      symbol: "ESH9"
+      symbol: "ESZ6"
+      symbol: "ESH0"
+      symbol: "ESM6"
+      symbol: "ESZ7"
+      symbol: "ESH6"
+      symbol: "ESM7"
+      symbol: "ESZ8"
+      symbol: "ESH7"
+      symbol: "ESH8"
+      symbol: "ESU6"
+      symbol: "ESZ5"
+      symbol: "ESZ9"
+      symbol: "E1D"
+      symbol: "ESZ5"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}

src/app_sparkd/sparkd-entrypoint.sh

@@ -4,9 +4,12 @@ init_timezone() {
   tz="${1:-$TZ}"
   zoneinfo_dir=/usr/share/zoneinfo
   if [ -f "$zoneinfo_dir/$tz" -a "$(readlink -f /etc/localtime)" != "$zoneinfo_dir/$tz" ]; then
-    echo "Initializing locales and timezone"
-    ln -snf "$zoneinfo_dir/$tz" /etc/localtime
-    sudo echo $tz > /etc/timezone
+    echo "Setting locales and timezone to $tz"
+    if [ -e /etc/localtime ]; then
+      sudo -n rm -f /etc/localtime
+    fi
+    sudo -n ln -snf "$zoneinfo_dir/$tz" /etc/localtime
+    echo $tz | sudo -n tee /etc/timezone > /dev/null
   fi
 }
 
@@ -36,10 +39,17 @@ roll_log() {
   fi
 }
 
+# Acquire sudo permission once for the script (needed for a few calls)
+echo bts | sudo -S -v
 data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
-echo bts | sudo -S chown -R bts:bts "$data_root"
+sudo -n chown -R bts:bts "$data_root"
 load_env "$data_root/conf/.env.local"
 init_timezone "$TZ"
 roll_log "$data_root/data/logging.sparkd/log"
 symlink "$data_root/conf/plugins" /home/bts/bin/plugins
+
+if  [ -e "/md_recording" -a ! -e "/home/bts/.local/share/bts/sparkd/data/md_recording" ]; then
+  symlink "/md_recording" /home/bts/.local/share/bts/sparkd/data/md_recording
+fi
+
 bash -c "${@:-./sparkd run}"

src/app_sparkd/sparkd/cmds/Run.cpp

@@ -10,7 +10,7 @@
 #include "spark/md/MBOServiceServer.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/SyntheticSecurityService.h"
-#include "spark/md/relay/query_server.h"
+#include "spark/md/relay/QueryServer.h"
 #include "spark/md/service/MarketDataServiceServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"

src/app_sparkd/sparkd/cmds/Web.cpp

@@ -6,7 +6,7 @@
 #include "spark/config/protos/quickbus_cfg.pb.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/SyntheticSecurityService.h"
-#include "spark/md/relay/query_server.h"
+#include "spark/md/relay/QueryServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"
 #include "spark/quickbus/core/Quickbus.h"

src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp

@@ -109,7 +109,8 @@ struct ExecutionGatewayProvider::PImpl {
          return make_gw<ICEExecutionGateway>(conn, conn, *l, o);
       case api::VENUE_GLBX:
          if (conn.use_ilink3()) {
-            make_gw<ilink3::GLBXiLink3ExecutionGateway>(conn, conn, *l, o);
+            return make_gw<ilink3::GLBXiLink3ExecutionGateway>(
+                conn, conn, *l, o);
          }
          return make_gw<GLBXExecutionGateway>(conn, conn, *l, o);
       case api::VENUE_DERIBIT:

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.cpp

@@ -21,11 +21,10 @@ using bts::domain::SecurityMarketDataStatistics;
 using bts::io::JsonReader;
 using std::map;
 using std::string;
-using std::string_view;
 using std::vector;
 
 static bool parse(
-    const map<string_view, string> &name_values,
+    const map<string, string> &name_values,
     vector<BinanceMiniTickerUpdate> &ticker_upds) {
    auto it = name_values.find(CCAPI_HTTP_BODY);
    if (it == name_values.end()) {

src/app_sparkd/sparkd/md/binance/BinancePriceStatistics.cpp

@@ -21,11 +21,10 @@ using bts::domain::SecurityMarketDataStatistics;
 using bts::io::JsonReader;
 using std::map;
 using std::string;
-using std::string_view;
 using std::vector;
 
 static bool parse(
-    const map<string_view, string> &name_values,
+    const map<string, string> &name_values,
     vector<BinancePriceStatistics> &result) {
    CCApiReader ccapi_reader{&name_values};
    nlohmann::json js;

src/app_sparkd/sparkd/md/binance/BinanceSecurityRequestor.cpp

@@ -180,7 +180,7 @@ struct BinanceSecurityRequestor::PImpl {
       for (const auto *msg : msgs) {
          bts_assert_msgf(
              msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST,
-             "Received msg w/ unexpected type:\n{}", msg->toPrettyString());
+             "Received msg w/ unexpected type:\n{}", msg->toStringPretty());
          parse_secs(*msg, types, added_secs);
       }
       auto added_sec_descs = get_sec_descs(added_secs);
@@ -214,7 +214,7 @@ struct BinanceSecurityRequestor::PImpl {
          _FATAL_MSGF(
              "on_exchange_info. {} event was unexpected:\n{}",
              ccapi::Event::typeToString(event.getType()),
-             event.toPrettyString());
+             event.toStringPretty());
          break;
       }
       return true;

src/app_sparkd/sparkd/md/binance_coin_futures/binance_coin_futures_price_statistics.cpp

@@ -21,11 +21,10 @@ using std::map;
 using std::pair;
 using std::size_t;
 using std::string;
-using std::string_view;
 using std::vector;
 
 static bool parse(
-    const map<string_view, string> &name_values,
+    const map<string, string> &name_values,
     vector<BinanceCoinFuturesPriceStatistics> &result) {
    CCApiReader ccapi_reader{&name_values};
    nlohmann::json js;

src/app_sparkd/sparkd/md/binance_coin_futures/binance_coin_futures_price_statistics_adaptor.cpp

@@ -60,7 +60,7 @@ void BinanceCoinFuturesPriceStatisticsAdaptor::on_messages(
          const auto n_total = n_hits + n_misses;
          _ERROR(
              "on_messages. Parse failed on {}/{} elements:\n{}", n_misses,
-             n_total, event.toPrettyString());
+             n_total, event.toStringPretty());
          continue;
       }
       for (const auto &bsr : binance_stats_recs) {

src/app_sparkd/sparkd/md/binance_coin_futures/binance_coin_futures_security_requestor.cpp

@@ -177,7 +177,7 @@ struct BinanceCoinFuturesSecurityRequestor::PImpl {
       for (const auto *msg : msgs) {
          bts_assert_msgf(
              msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST,
-             "Received msg w/ unexpected type:\n{}", msg->toPrettyString());
+             "Received msg w/ unexpected type:\n{}", msg->toStringPretty());
          parse_secs(*msg, added_secs);
       }
       auto added_sec_descs = get_sec_descs(added_secs);
@@ -211,7 +211,7 @@ struct BinanceCoinFuturesSecurityRequestor::PImpl {
          _FATAL_MSGF(
              "on_exchange_info. {} event was unexpected:\n{}",
              ccapi::Event::typeToString(event.getType()),
-             event.toPrettyString());
+             event.toStringPretty());
          break;
       }
       return true;

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatistics.cpp

@@ -23,11 +23,10 @@ using std::int64_t;
 using std::map;
 using std::ostream;
 using std::string;
-using std::string_view;
 using std::vector;
 
 static bool parse(
-    const map<string_view, string> &name_values,
+    const map<string, string> &name_values,
     vector<BinanceUsPriceStatistics> &result) {
    CCApiReader ccapi_reader{&name_values};
    nlohmann::json js;

src/app_sparkd/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp

@@ -69,10 +69,10 @@ static vector<string> get_sec_descs(const vector<Security> &sec_ptrs) {
 static void validate_response(const ccapi::Event &event, bool is_expected) {
    bts_assert_msgf(
        event.getType() == ccapi::Event::Type::RESPONSE,
-       "Received event w/ unexpected type:\n{}", event.toPrettyString());
+       "Received event w/ unexpected type:\n{}", event.toStringPretty());
    bts_assert_msgf(
        is_expected, "Received unexpected response from Binance US:\n{}",
-       event.toPrettyString());
+       event.toStringPretty());
 }
 
 struct BinanceUsSecurityRequestor::PImpl {
@@ -181,7 +181,7 @@ struct BinanceUsSecurityRequestor::PImpl {
       for (const auto *msg : msgs) {
          bts_assert_msgf(
              msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST,
-             "Received msg w/ unexpected type:\n{}", msg->toPrettyString());
+             "Received msg w/ unexpected type:\n{}", msg->toStringPretty());
          parse_secs<T>(*msg, types, added_secs);
       }
       auto added_sec_descs = get_sec_descs(added_secs);
@@ -200,7 +200,7 @@ struct BinanceUsSecurityRequestor::PImpl {
        const CCApiCorrelationID & /*corr_id*/) {
       validate_response(event, exch_info_req_pending);
       LOG(md, Diag, "Received Binance US exchange info (spots):\n{}",
-          event.toPrettyString());
+          event.toStringPretty());
       update_securities<BinanceUsSpot>(msgs, {SEC_TYPE(SPOT)});
       exch_info_req_pending = false;
       if (external_req_pending && !is_busy()) {

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.cpp

@@ -21,11 +21,10 @@ using std::map;
 using std::pair;
 using std::size_t;
 using std::string;
-using std::string_view;
 using std::vector;
 
 static bool parse(
-    const map<string_view, string> &name_values,
+    const map<string, string> &name_values,
     vector<BinanceUsdsFuturesPriceStatistics> &result) {
    CCApiReader ccapi_reader{&name_values};
    nlohmann::json js;

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatisticsAdaptor.cpp

@@ -60,7 +60,7 @@ void BinanceUsdsFuturesPriceStatisticsAdaptor::on_messages(
          const auto n_total = n_hits + n_misses;
          _ERROR(
              "on_messages. Parse failed on {}/{} elements:\n{}", n_misses,
-             n_total, event.toPrettyString());
+             n_total, event.toStringPretty());
          continue;
       }
       for (const auto &bsr : binance_stats_recs) {

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp

@@ -178,7 +178,7 @@ struct BinanceUsdsFuturesSecurityRequestor::PImpl {
       for (const auto *msg : msgs) {
          bts_assert_msgf(
              msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST,
-             "Received msg w/ unexpected type:\n{}", msg->toPrettyString());
+             "Received msg w/ unexpected type:\n{}", msg->toStringPretty());
          parse_secs(*msg, added_secs);
       }
       auto added_sec_descs = get_sec_descs(added_secs);
@@ -212,7 +212,7 @@ struct BinanceUsdsFuturesSecurityRequestor::PImpl {
          _FATAL_MSGF(
              "on_exchange_info. {} event was unexpected:\n{}",
              ccapi::Event::typeToString(event.getType()),
-             event.toPrettyString());
+             event.toStringPretty());
          break;
       }
       return true;