Add scaffolding for PandasExpressions

Improves performance compared to accessing columns by name

Spark.Api.Server.sln

@@ -14,36 +14,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -100,102 +80,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
@@ -204,6 +88,38 @@ Global
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
 		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -214,19 +130,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
-		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
-		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
-	EndGlobalSection
 EndGlobal

src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs

@@ -4,8 +4,6 @@ namespace bts.SilverPine {
    using System;
 
    public class NullFilterStream : ViewStream {
-      public NullFilterStream(ViewStream parent_str, string col_name)
-      : this(parent_str, col_name, true) { }
       public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
          ViewStream rs = new FilterStream(parent_str,
          r => {

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -1,91 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-
-   using ZstdSharp.Unsafe;
-
-   public enum GroupByBundleOp {
-      Mean,
-      Median,
-      Mode,
-      StdDev,
-      Max,
-      Min,
-      First,
-      Last,
-      Sum
-   }
-   public class GroupByBundle {
-      public readonly List<Tuple<string, GroupByBundleOp>> operations;
-
-      public static GroupByBundle parse(string text) {
-         // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
-         string[] calcs = text.Split(',');
-         foreach (string c in calcs) {
-            string[] toks = c.Split(':');
-            string inp_col = toks[0].Trim();
-            string op_txt = toks[1].Trim();
-            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
-            rv.Add(Tuple.Create(inp_col, op));
-         }
-         return new GroupByBundle(rv);
-      }
-
-      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
-         operations = operations_;
-      }
-
-      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
-         if (rows.Count() == 0) {
-            return null;
-         }
-
-         IBlueRow ir = rows.ElementAt(0);
-         IBlueTableSchema in_schema = ir.schema;
-         IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
-            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
-         foreach (var t in parameters.operations) {
-            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
-            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
-            out_col_map[out_col_name] = t;
-         }
-         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
-         foreach (var kv in out_col_map) {
-            string out_col_name = kv.Key;
-            string in_col_name = kv.Value.Item1;
-            GroupByBundleOp op = kv.Value.Item2;
-            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
-            double comp_res = compute_op(col_vals, op);
-            or.setValue(out_col_name, comp_res);
-         }
-         return or;
-      }
-
-      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
-         switch (op) {
-            case GroupByBundleOp.Max:
-               return col_vals.Max<double>();
-            case GroupByBundleOp.Min:
-               return col_vals.Min<double>();
-            case GroupByBundleOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
-            case GroupByBundleOp.Sum:
-               return col_vals.Sum();
-            case GroupByBundleOp.First:
-               return col_vals.First();
-            case GroupByBundleOp.Last:
-               return col_vals.Last();
-            default:
-               // TODO
-               Log.assert(false);
-               return Double.NaN;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -2,13 +2,10 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.Json;
 
 using bts;
-using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("aggregate")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;
@@ -46,31 +43,13 @@ namespace BCG {
          _main_stream = main_stream;
          _groupby_func = groupby_func;
          _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
       }
 
       public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
-         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
-         Func<string[]> groupby_cols_func = () => {
-            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
-         };
-         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
-            if (!String.IsNullOrEmpty(optional_sort_col))
-               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
-            return new IBlueRow[] { rv }.ToList();
-         };
-         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
-         mySetRealSource(rs);
-      }
-      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
-      this(parent_str, String.Empty, standard_group_by_text) {
-      }
-
 
       protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
 
@@ -104,9 +83,7 @@ namespace BCG {
             Log.DataFlow.info(
                $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
          }
-         // TODO,
-         if (total_state != null)
-            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -1,12 +1,10 @@
 using bts;
-using bts.collections;
 
 using mbts;
 
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Security.Cryptography;
 
 namespace BCG {
@@ -37,38 +35,6 @@ namespace BCG {
          _is_tag_along = _join_spec.join_col_names.Length == 0;
       }
 
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
       public void init() {
          _join_spec.stream.onChange += _on_stream_cb;
       }
@@ -77,7 +43,6 @@ namespace BCG {
          Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
          if (_schema == null) {
             _schema = dp.schema;
             _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs

@@ -7,12 +7,11 @@ using System.Threading.Tasks;
 using mbts;
 
 namespace BCG {
-   // this a collection of data, theres no joining logic in here
    public class NonIncJoinSpec {
       public ViewStream stream;
       // join col names are the names on which the streams are joined (for equality),
       // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // projected col naes are their new names, ie if its x:l_x, then x is in the 
       // data_col_names and l_x in projected cols
       public string[] join_col_names, data_col_names, projected_col_names;
 

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -1,8 +1,9 @@
+using Google.Protobuf.WellKnownTypes;
+
 using System;
+using BCG.Utilities.General;
 
-using BCG;
-
-namespace bts.SilverPine {
+namespace BCG {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -9,16 +9,161 @@ using BCG;
 using System.Linq;
 using System.Data.Entity;
 using System.Collections.Generic;
-using System.Data.SQLite;
-
 
 namespace bts.SilverPine {
 
-   public static class SQLiteHelper {
-      public static string location = "C:\\Users\\m_tah\\development\\data\\";
-      public static string db_name = "stream_data.db";
+   public class DataRowWrapperRow : IBlueRow {
+      private readonly DataRow _row;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _id;
+      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
+         _row = row;
+         _id = id;
+         _schema = schema;
+      }
+      public virtual IBlueTableSchema schema { get { return _schema; } }
 
-      public static DataTable execute_select_query(string connection_string, string query) {
+      public virtual object this[string col_name] {
+         get {
+            ColumnInfo ci = _schema[col_name];
+            return this[ci.colNo];
+         }
+      }
+      public virtual object this[int index] {
+         get {
+            if (_schema[index].colName == "id")
+               return _id;
+            return _row[index];
+         }
+      }
+
+      public virtual int Count { get { return _row.ItemArray.Length; } }
+
+      public virtual bool IsSynchronized { get { return false; } }
+
+      public virtual object SyncRoot { get { return this; } }
+
+      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
+
+      public virtual bool IsFixedSize { get { return true; } }
+
+      virtual public bool IsReadOnly { get { return true; } }
+
+      /// <summary>
+      /// The key can either be the string corrsponding to the column name or
+      /// the order of the column
+      /// </summary>
+      public virtual object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            else if (key is int)
+               return this[(int)key];
+            throw new BlueDataException("Wrong type while indexing the row");
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public virtual ICollection Keys { get { return _schema.columnNames; } }
+
+      public virtual ICollection Values { get { return _row.ItemArray; } }
+
+      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
+
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+   }
+
+   public class DataTableWrapperTable : IBlueRowCollection {
+
+      public class DataTableWrapperTableEnumerator : IEnumerator {
+         private IEnumerator _enumerator;
+         private IBlueTableSchema _schema;
+         private int _id = 0;
+         public DataTableWrapperTableEnumerator(DataTable table) {
+            _enumerator = table.Rows.GetEnumerator();
+            _schema = BlueDataUtility.getSchema(table);
+
+         }
+         public object Current {
+            get {
+               DataRow r = _enumerator.Current as DataRow;
+               return new DataRowWrapperRow(_schema, r, _id);
+            }
+         }
+         public bool MoveNext() {
+            _id++;
+            return _enumerator.MoveNext();
+         }
+         public void Reset() {
+            _id = 0;
+            _enumerator.Reset();
+         }
+      }
+
+      private readonly DataTable _table;
+      private IBlueTableSchema _schema;
+      private IEnumerator _enumerator;
+
+      public DataTableWrapperTable(DataTable table) {
+         _table = table;
+         _schema = BlueDataUtility.getSchema(_table);
+         _enumerator = new DataTableWrapperTableEnumerator(table);
+      }
+      public virtual IBlueRow this[int id] {
+         get {
+            // its assumed the rows index == id
+            DataRow dr = _table.Rows[id];
+            return new DataRowWrapperRow(_schema, dr, id);
+         }
+         set {
+
+         }
+      }
+
+      public virtual IBlueTableSchema schema { get { return _schema; } }
+
+      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
+
+      // private List<IBlueRow> _rows() {
+      //    List<IBlueRow> rv = new List<IBlueRow>();
+      //    int id = 0;
+      //    foreach (DataRow r in _table.Rows) {
+      //       rv.Add(new DataRowWrapperRow(r, id++));
+      //    }
+      //    return rv;
+      // }
+
+      public virtual IEnumerator GetEnumerator() {
+         // var rows = _rows();
+         // return rows.GetEnumerator();
+         return _enumerator;
+      }
+
+      public virtual bool IsSynchronized { get { return false; } }
+
+      public virtual object SyncRoot { get { return this; } }
+
+      public virtual void CopyTo(Array array, int index) {
+         Log.assert(false);
+         //_rows().CopyTo(array as IBlueRow[], index); 
+      }
+
+      public virtual int Count {
+         get { return _table.Rows.Count; }
+      }
+   }
+
+   public static class SQLiteHelper {
+      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
          // Create a new DataTable to store the results
          DataTable dataTable = new DataTable();
 
@@ -38,85 +183,48 @@ namespace bts.SilverPine {
                Console.WriteLine("An error occurred: " + ex.Message);
             }
          }
+
          // Return the populated DataTable
          return dataTable;
       }
+   }
 
-      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
-
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-
-            // Begin a transaction to optimize read speed
-            using (var transaction = connection.BeginTransaction())
-            using (var command = new SQLiteCommand(query, connection, transaction))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-
-                  result.Add(row);
-               }
-
-               // Commit the transaction
-               transaction.Commit();
-            }
-         }
-
-         return result;
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+      public SQLiteStream(string connection_string, string query) {
+         _connection_string = connection_string;
+         _query = query;
       }
 
-      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
-         var result = new List<Dictionary<string, object>>();
 
-         using (var connection = new SQLiteConnection(connectionString)) {
-            try {
-               connection.Open();
-
-               using (var command = new SQLiteCommand(query, connection))
-               using (var reader = command.ExecuteReader()) {
-                  while (reader.Read()) {
-                     var row = new Dictionary<string, object>();
-
-                     for (int i = 0; i < reader.FieldCount; i++) {
-                        string columnName = reader.GetName(i);
-                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                        row[columnName] = columnValue;
-                     }
-
-                     result.Add(row);
-                  }
-               }
-            } catch (Exception ex) {
-               Console.WriteLine("An error occurred: " + ex.Message);
-            }
-         }
-         return result;
+      private void _read_from_sqlite_nd_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Start();
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
       }
 
-      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
-         using (var connection = new SQLiteConnection(connectionString)) {
-            connection.Open();
-            using (var command = new SQLiteCommand(query, connection))
-            using (var reader = command.ExecuteReader()) {
-               while (reader.Read()) {
-                  var row = new Dictionary<string, object>();
-                  for (int i = 0; i < reader.FieldCount; i++) {
-                     string columnName = reader.GetName(i);
-                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
-                     row[columnName] = columnValue;
-                  }
-                  yield return row;
-               }
-            }
-         }
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_nd_fire();
       }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
    }
 }
 

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -1,73 +0,0 @@
-namespace bts.SilverPine {
-   using System;
-   using System.Data;
-   using System.Data.SQLite;
-   using System.Diagnostics;
-   using System.Collections;
-
-
-   using BCG;
-
-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-
-      private SQLiteStream(string connection_string, string query, bool noop) {
-         _connection_string = connection_string;
-         _query = query;
-      }
-
-      public SQLiteStream(string db_name, string query)
-      :
-      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
-
-      public SQLiteStream(string query)
-      :
-      this(SQLiteHelper.db_name, query) {
-      }
-
-
-      private void _read_from_sqlite_and_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Restart();
-         #region perf measures
-         // SQLiteHelper.get_table(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         // SQLiteHelper.get_table_2(_connection_string, _query);
-         // _stopwatch.Stop();
-         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         // _stopwatch.Restart();
-         #endregion
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_and_fire();
-      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
-   }
-
-}

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -80,7 +80,7 @@ namespace BCG {
 
       public IBlueRow map(IBlueRow row) {
          MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
          return rv;
          //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
       }

src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs

@@ -10,11 +10,9 @@ namespace bts.SilverPine {
          Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
             IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
             rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
-            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
             return rv;
          };
          ViewStream rs = new MapStream(parent_stream,
@@ -25,11 +23,9 @@ namespace bts.SilverPine {
             MutableBlueRow rv = new MutableBlueRow(s, r);
             rv.setValue($"{inp_col}_date", dt.Date);
             rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_day_of_month", dt.Day);
-            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
             rv.setValue($"{inp_col}_hour", dt.Hour);
             rv.setValue($"{inp_col}_min", dt.Minute);
-            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
             return rv;
          });
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs

@@ -39,7 +39,7 @@ namespace bts.SilverPine {
          mySetRealSource(rs);
       }
 
-      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
       // we should generlize this for any math operation
       public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
          // use GlobalMapStream
@@ -91,32 +91,19 @@ namespace bts.SilverPine {
          DataPacket dp = dp_obj as DataPacket;
          List<IBlueRow> rows = dp.newRows.to_list();
          var f1_rows = rows.Where(r => _f1(r));
-         //Log.assert(f1_rows.Count() == 1);
+         Log.assert(f1_rows.Count() == 1);
          var f2_rows = rows.Where(r => _f2(r));
-         //Log.assert(f2_rows.Count() == 1);
-         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
-            IBlueRow f1_r = f1_rows.ElementAt(0);
-            IBlueRow f2_r = f2_rows.ElementAt(0);
-            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         } else {
-            IBlueRow f1_r = rows.ElementAt(0);
-            IBlueRow f2_r = rows.ElementAt(0);
-            double diff = Double.NaN;
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         }
+         Log.assert(f2_rows.Count() == 1);
+         IBlueRow f1_r = f1_rows.ElementAt(0);
+         IBlueRow f2_r = f2_rows.ElementAt(0);
+         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+         BlueTable out_table = new BlueTable(out_schema);
+         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+         out_row.setValue("diff", diff);
+         out_table.addRow(out_row);
+         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
 
       }
    }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -18,7 +18,6 @@ using System.Text.Json.Nodes;
 using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
-using bts;
 using Spark.Core;
 using System.Numerics;
 
@@ -336,10 +335,6 @@ namespace BCG {
          return rv;
       }
 
-      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
-         return input_schema.columnNames.Contains(col);
-      }
-
       public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
          List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
          foreach (Tuple<Type, string> c in cols)
@@ -1852,10 +1847,8 @@ namespace BCG {
 
       public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
          List<int> rv = new List<int>();
-         foreach (string c in col_names) {
-            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
+         foreach (string c in col_names)
             rv.Add(schema[c].colNo);
-         }
          return rv.ToArray();
       }
 
@@ -2342,25 +2335,17 @@ namespace BCG {
          try {
             if (v == null || v == DBNull.Value)
                return error_val;
-
-            if (v is DateTime) {
-               return (DateTime)v;
-            } else if (v is string) {
-               DateTime rv_dt;
-
-               // Try parsing the string as a DateTime
-               if (DateTime.TryParse((string)v,
-                                     System.Globalization.CultureInfo.InvariantCulture,
-                                     System.Globalization.DateTimeStyles.RoundtripKind,
-                                     out rv_dt)) {
+            else {
+               if (v is DateTime)
+                  return (DateTime)v;
+               else if (v is string) {
+                  DateTime rv_dt = error_val;
+                  DateTime.TryParse((string)v, out rv_dt);
                   return rv_dt;
                } else {
-                  return error_val; // Return the error value if parsing fails
+                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+                  return DateTime.FromOADate(oa_date);
                }
-            } else {
-               // Assume the input can be converted to a double and use FromOADate
-               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-               return DateTime.FromOADate(oa_date);
             }
          } catch {
             return error_val;

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -53,11 +53,6 @@ namespace BCG {
       /// </summary>
       /// <param name="assemblyName"></param>
       public void loadAssembly(string assemblyName, Func<Type, bool> typeFilter = null) {
-         Func<Type, bool> is_silverpine_only = (t) => {
-            if (!assemblyName.Contains("SharedStreams.dll"))
-               return true;
-            return t.Namespace == "bts.SilverPine";
-         };
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
             return;
@@ -71,7 +66,7 @@ namespace BCG {
             types = types.Where(typeFilter);
          }
          foreach (Type type in types)
-            if (!type.IsInterface && is_silverpine_only(type)
+            if (!type.IsInterface
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
                && type.IsPublic) {

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -24,11 +24,12 @@ namespace bts {
       };
 
       public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
-         Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} tag_handler: {tag_handler}");
+         bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
          this.tag = tag;
          this.tag_type = tag_type;
          _tag_handler = tag_handler;
          Log.notnull(_tag_handler);
+         Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} _tag_handler: {_tag_handler}");
       }
 
       public string tag { get; }

src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs

@@ -9,90 +9,8 @@ namespace bts.SilverPine {
    using ZstdSharp.Unsafe;
 
    public static class ContractUtils {
-      public class FutureTradableContract {
-         public string symbol;
-         public DateTime expiration;
-         public Exchange exchange;
-         public string contract_text;
-      }
-      public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
-         public string symbol {
-            get {
-               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
-            }
-         }
-         public string contract_text;
-      }
-      public class FutureContinousContract {
-         public string symbol;
-         public Exchange exchange;
-      }
-      // this is for all future conytracts belonging to a symbol
-      public class FutureWildcardContract {
-         public string symbol;
-         public Exchange exchange;
-      }
-      public class FutureContract {
-         public FutureWildcardContract wildcard_contract = null;
-         public FutureTradableContract future_tradable_contract = null;
-         public FutureSpreadContract spread_tradable_contract = null;
-         public FutureContinousContract continous_constract = null;
-
-         public FutureContract(string txt) {
-            wildcard_contract = parse_wildcard_contract(txt);
-            if (wildcard_contract != null)
-               return;
-            FutureTradableContract fc = parse_future_contract(txt);
-            if (fc != null) {
-               future_tradable_contract = fc;
-               return;
-            }
-            FutureSpreadContract sc = parse_future_spread_contract(txt);
-            if (sc != null) {
-               spread_tradable_contract = sc;
-               return;
-            }
-            // this is not totally true, since ParseFutureSpreadContract does not
-            // parse ICS spreads
-            continous_constract = new FutureContinousContract() { symbol = txt };
-         }
-         public bool is_continous_contract => continous_constract != null;
-         public string symbol() {
-            if (wildcard_contract != null)
-               return wildcard_contract.symbol;
-            if (continous_constract != null) {
-               return continous_constract.symbol;
-            } else if (future_tradable_contract != null) {
-               return future_tradable_contract.symbol;
-            } else if (spread_tradable_contract != null) {
-               return spread_tradable_contract.symbol;
-            }
-            Log.assert(false);
-            return String.Empty;
-         }
-         public string contract_text {
-            get {
-               if (future_tradable_contract != null)
-                  return future_tradable_contract.contract_text;
-               if (spread_tradable_contract != null)
-                  return spread_tradable_contract.contract_text;
-               return null;
-            }
-         }
-      }
       #region Parsing
-      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
-         if (contract_txt.EndsWith("*")) {
-            contract_txt = contract_txt.Replace("*", String.Empty);
-            return new FutureWildcardContract() {
-               symbol = contract_txt,
-               exchange = exchange
-            };
-         }
-         return null;
-      }
-      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+      public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) {
          // Validate the future string length
          if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
             return null;
@@ -138,24 +56,23 @@ namespace bts.SilverPine {
 
          // Create the FutureContract instance and set its properties
          DateTime expiration = new DateTime(year, month, 1);
-         var futureContract = new FutureTradableContract {
+         var futureContract = new FutureContract {
             symbol = symbol,
             expiration = expiration,
-            exchange = exchange,
-            contract_text = contract_txt
+            exchange = exchange
          };
 
          return futureContract;
       }
       // Method to parse a futures spread, e.g. "CLX4-CLZ4"
-      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+      public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) {
          // Split the string by '-' to handle each leg
-         var spreadParts = contract_txt.Split('-');
+         var spreadParts = spreadString.Split('-');
          if (spreadParts.Length < 2) {
             return null; // Not a valid spread format
          }
 
-         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureContract>>() };
 
          foreach (var part in spreadParts) {
             // Split each leg by '*', assuming the format is weight*future or just future
@@ -175,32 +92,33 @@ namespace bts.SilverPine {
             }
 
             // Parse the future contract
-            var futureContract = parse_future_contract(futurePart, exchange);
+            var futureContract = ParseFutureContract(futurePart, exchange);
             if (futureContract == null) {
                return null; // Failed to parse the future part
             }
 
             // Add the leg to the spread contract
-            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
+            spreadContract.legs.Add(new Tuple<double, FutureContract>(weight, futureContract));
          }
+
          return spreadContract;
       }
 
-      // public static ContractType DetermineFutureOrSpread(string contract) {
-      //    if (contract.Contains("-")) {
-      //       return ContractType.FutureSpread;
-      //    } else {
-      //       return ContractType.Future;
-      //    }
-      // }
+      public static ContractType DetermineFutureOrSpread(string contract) {
+         if (contract.Contains("-")) {
+            return ContractType.FutureSpread;
+         } else {
+            return ContractType.Future;
+         }
+      }
       #endregion
 
       public static ContractType get_type(string contract_txt) {
-         FutureTradableContract fc = parse_future_contract(contract_txt);
+         FutureContract fc = ParseFutureContract(contract_txt);
          if (fc != null) {
             return ContractType.Future;
          }
-         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
+         FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt);
          if (sc != null) {
             return ContractType.FutureSpread;
          }
@@ -215,11 +133,11 @@ namespace bts.SilverPine {
       }
 
       public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
-         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
+         ContractUtils.FutureContract fc = ParseFutureContract(symbol);
          if (fc != null) {
             return Tuple.Create(ContractCategory.FutureString, fc.symbol);
          }
-         var sc = parse_future_spread_contract(symbol);
+         var sc = ParseFutureSpreadContract(symbol);
          if (sc != null) {
             return Tuple.Create(ContractCategory.FutureString, sc.symbol);
          }
@@ -247,6 +165,18 @@ namespace bts.SilverPine {
       public enum Exchange {
          CME
       }
-
+      public class FutureContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+      }
    }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -1,107 +1,36 @@
-namespace bts.SilverPine {
+namespace BCG {
    using BCG.Utilities.General;
 
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
    using bts.md;
-   using BCG;
+   using bts.spark;
+   using bts.utils;
 
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
    using Spark.Core.Disposables;
    using Spark.Core.Reactive;
 
    using System;
+   using System.Collections.Concurrent;
    using System.Collections.Generic;
+   using System.Collections.ObjectModel;
    using System.Linq;
    using System.Reactive.Linq;
    using bts.api.core;
    using Spark.Api.MarketData;
 
    using SecurityKey = System.UInt32;
-   using System.Data.Entity.Infrastructure;
-
-   public class FakeBBOStream : ViewStream, IWorker {
-      private string _symbol;
-      public BlueTable _table = new BlueTable(BBOStream._bbo_schema);
-      private DateTime _last_fired_time = DateTime.MinValue;
-      private TimeSpan _period = TimeSpan.FromMilliseconds(333);
-      private double _start_price;
-      private double _inc;
-      private int _total_count = 5;
-      private Random _rnd = new Random(DateTime.Now.Millisecond);
-      private string[] _months = new string[]{
-         "F","G", "H", "J","K", "M", "N","Q","U","V","X","Z"
-      };
-      private ViewStreamDatabase _vsd;
-      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc)
-      :
-      this(contract_type, exchange, symbol, start_price, inc, ViewStreamDatabase.instance) { }
-
-      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc, ViewStreamDatabase vsd) {
-         _symbol = symbol;
-         _start_price = start_price;
-         _inc = inc;
-         _vsd = vsd;
-         Log.assert(contract_type.ToLower() == "future");
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-      }
-
-      private bool _random_truefase() {
-         int r = _rnd.Next(0, 100);
-         return r % 2 == 0;
-      }
-
-      private IBlueRow _perturb_row(IBlueRow er) {
-         MutableBlueRow rv = new MutableBlueRow(er);
-         double bid_price = rv.get<double>("bid_price");
-         bid_price += _inc;
-         double ask_price = bid_price + _inc;
-         rv.setValue("bid_price", bid_price);
-         rv.setValue("ask_price", ask_price);
-         return rv;
-      }
-
-      public void doWork() {
-         if (DateTime.Now - _last_fired_time < _period)
-            return;
-         _last_fired_time = DateTime.Now;
-         if (_table.Count == 0) {
-            // first fire_symbol
-            string year_code = DateTime.Now.Year.ToString().Last().ToString();
-            for (int i = 0; i < _total_count; i++) {
-               string month_code = _months[i % _months.Length];
-               string sec = $"{_symbol}{month_code}{year_code}";
-               double bid_price = _start_price + _rnd.Next(-10, 10) * _inc;
-               double ask_price = bid_price + _inc;
-               BlueRow nr = new BlueRow(_table.schema, _table.Count, _table.Count, sec,
-                                       bid_price, _rnd.Next(1, 10),
-                                       ask_price, _rnd.Next(1, 10));
-               _table.addRow(nr);
-               var dp = BlueDataUtility.build_dp(new BlueTable(_table), myLastValue as DataPacket);
-               myFireEvent(dp);
-            }
-         } else {
-            BlueTable deleted_rows = new BlueTable(_table.schema);
-            BlueTable added_rows = new BlueTable(_table.schema);
-            // go through all rows, select some at random, add them to deleted, get
-            // the replaced one and add it to added, updates table state and fire
-            for (int i = 0; i < _table.Count; i++) {
-               if (_random_truefase()) {
-                  IBlueRow er = _table[i];
-                  IBlueRow nr = _perturb_row(er);
-                  _table.deleteRow(er);
-                  _table.addRow(nr);
-               }
-            }
-            var dp = BlueDataUtility.build_dp_inc(new BlueTable(_table), myLastValue as DataPacket, true);
-            myFireEvent(dp);
-         }
-      }
-   }
+   using System.Data;
+   using System.Reflection.Metadata;
+   using bts;
 
    [DataStreamUserRespresentation("bbo")]
    public class BBOStream : ViewStream, IWorker {
@@ -111,7 +40,7 @@ namespace bts.SilverPine {
       private int _id = 0;
       IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
       MarketDataClient _md_client;
-      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
@@ -127,11 +56,9 @@ namespace bts.SilverPine {
       }
 
       public BBOStream(ViewStreamDatabase vsd) {
-         // bts.Log.Gen.info("BBOStream.cons");
-         // _vsd = vsd;
-         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         mySetRealSource(rs);
+         bts.Log.Gen.info("BBOStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
       }
 
       protected override void myOnFirstSubscriber() {

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs

@@ -1,241 +0,0 @@
-namespace bts.SilverPine {
-   using BCG;
-
-   using System.Runtime.CompilerServices;
-
-   using ZstdSharp.Unsafe;
-
-   public class DateTimeRange {
-      public readonly DateTime range_beg;
-      public readonly DateTime range_end;
-      public DateTimeRange(DateTime range_beg_, DateTime range_end_) {
-         range_beg = range_beg_;
-         range_end = range_end_;
-      }
-      public static DateTimeRange parse(string range_beg_, string range_end_) {
-         DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_);
-         DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_);
-         return new DateTimeRange(range_beg, range_end);
-      }
-   }
-
-   public class FutureBarsLoader {
-      public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
-         switch (bar_size) {
-            case BarSize.ONE_DAY: {
-                  if (is_continous) {
-                     return $"bars_future_{symbol}_continous_1day";
-                  } else
-                     return $"bars_future_{symbol}_all_1day";
-               }
-            case BarSize.ONE_MIN: {
-                  if (is_continous) {
-                     return $"bars_future_{symbol}_continous_1min";
-                  } else
-                     return $"bars_future_{symbol}_all_1min";
-               }
-         }
-         Log.assert(false);
-         return null;
-      }
-   }
-   public enum BarSize {
-      ONE_DAY = 0,
-      ONE_MIN
-   }
-
-   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   public class FuturesHistorical1DayBarsStream : ViewStream {
-
-      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
-      public FuturesHistorical1DayBarsStream(string symbol) :
-      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
-
-      // creates the non raw stream (ie for tradable contract, or continous contract)
-      public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
-     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
-
-      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
-      // it gets the raw table with all contracts for that symbol
-      public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract,
-                                             DateTimeRange dt_range) {
-         DateTime beg = dt_range.range_beg;//.ToUniversalTime(),
-         DateTime end = dt_range.range_end;//.ToUniversalTime();
-         beg = beg.Date;
-         end = end.Date.Add(new TimeSpan(23, 59, 59));
-         // this returns rows from beg and end inclusive
-         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
-            string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
-               range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
-            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
-            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
-         };
-         string symbol = future_contract.symbol();
-         if (future_contract.wildcard_contract != null) {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
-            ViewStream rs = get_sqlite_str(table_name);
-            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
-            mySetRealSource(rs);
-            return;
-         }
-         if (future_contract.wildcard_contract != null) {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
-            ViewStream rs = get_sqlite_str(table_name);
-            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
-            mySetRealSource(rs);
-            return;
-         }
-         if (future_contract.is_continous_contract) {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
-            ViewStream rs = get_sqlite_str(table_name);
-            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
-            mySetRealSource(rs);
-            return;
-         } else {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
-            ViewStream rs = get_sqlite_str(table_name);
-            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
-            //TODO: filter it 
-            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
-            mySetRealSource(rs);
-         }
-      }
-   }
-
-   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   public class FuturesHistorical1MinuteBarsStream : ViewStream {
-      // same as the above but for one min
-      // this gets the raw bars for a symbol
-      public FuturesHistorical1MinuteBarsStream(string symbol) :
-      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
-
-      // creates the non raw stream (ie for tradable contract, or continous contract)
-      public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
-     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
-
-      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
-      // it gets the raw table with all contracts for that symbol
-      public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
-                                             DateTimeRange dt_range) {
-         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
-            string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
-               range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
-            //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
-            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
-            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
-         };
-         string symbol = future_contract.symbol();
-         if (future_contract.wildcard_contract != null) {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
-            ViewStream rs = get_sqlite_str(table_name);
-            mySetRealSource(rs);
-            return;
-         }
-         if (future_contract.is_continous_contract) {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
-            ViewStream rs = get_sqlite_str(table_name);
-            mySetRealSource(rs);
-            return;
-         } else {
-            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
-            ViewStream rs = get_sqlite_str(table_name);
-            // TODO: filter
-            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
-            mySetRealSource(rs);
-         }
-      }
-   }
-
-   [DataStreamUserRespresentation("FutureHistoricalBars")]
-   public class FutureHistoricalBarsStream : ViewStream {
-      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
-         DateTime dt_range_beg = DateTime.Parse(range_beg);
-         DateTime dt_range_end = DateTime.Parse(range_end);
-         TimeSpan diff = dt_range_end - dt_range_beg;
-         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY;
-         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
-         mySetRealSource(rs);
-      }
-      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
-                                        DateTime dt_range_beg, DateTime dt_range_end)
-                                        :
-      this(new ContractUtils.FutureContract(fut_contract), Enum.Parse<BarSize>(bar_size, true),
-         new DateTimeRange(dt_range_beg, dt_range_end)) { }
-
-      public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size,
-                                        DateTimeRange dt_range) {
-         switch (bar_size) {
-            case BarSize.ONE_DAY: {
-                  ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range);
-                  mySetRealSource(rs);
-                  break;
-               }
-            case BarSize.ONE_MIN: {
-                  ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range);
-                  mySetRealSource(rs);
-                  break;
-               }
-         }
-      }
-   }
-
-   [DataStreamUserRespresentation("FuturePriceDiff")]
-   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
-      }
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
-         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
-
-         rs = new ColumnDifferenceStream(rs, r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt1.Equals(r_dt);
-            return rv;
-         },
-         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
-         r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt2.Equals(r_dt);
-            return rv;
-         }, col, col);
-         rs = new NewProjectStream(rs, "diff !");
-         mySetRealSource(rs);
-      }
-   }
-
-   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
-   // the contract columns are a union, of all properties with some of them empty
-   public class ContractExpanderStream : ViewStream {
-      private static IBlueTableSchema map_schema(IBlueTableSchema schema) {
-         return schema.add_column(TypeCode.String, "contract_type");
-      }
-      public ContractExpanderStream(ViewStream parent_str, string contract_col_name) {
-         ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema),
-         r => {
-            string contract = r.get<string>(contract_col_name);
-            var ct = ContractUtils.get_type(contract);
-            MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r);
-            rv.setValue("contract_type", ct.ToString());
-            return rv;
-         });
-         mySetRealSource(rs);
-      }
-   }
-
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -1,46 +0,0 @@
-using BCG;
-
-namespace bts.SilverPine {
-   /*
-      ----------------------------------------------------------------------
-                                  *Expression*
-      ----------------------------------------------------------------------
-          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
-          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
-          r=contractExpander(r,"symbol")
-          r=filter(r,"contract_type='FutureSpread'")
-          r=aggregate(r,"symbol","volume:Sum","ts_event")
-          r=aggregate(r,"volume_sum:Max")
-          r=newProject(r,"volume_sum_max:volume symbol !")
-
-      ----------------------------------------------------------------------
-                                    *Schema*
-      ----------------------------------------------------------------------
-          symbol                          System.String
-          id                              System.Int32
-          volume                          System.Double
-   */
-
-   [DataStreamUserRespresentation("VolumeAgg")]
-   public class VolumeAggregationStream : ViewStream {
-      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
-      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
-
-      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
-      :
-      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
-
-      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
-         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
-         ViewStream r;
-         r = new FutureHistoricalBarsStream(future_contract,
-                                             range_beg.ToString(), range_end.ToString());
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, $"contract_type='{contract_type}'");
-         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
-         r = new NonIncGroupByStream(r, "volume_sum:Max");
-         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
-         mySetRealSource(r);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs

@@ -1,117 +1,68 @@
-// namespace bts.SilverPine {
-//    using bts;
-//    using BCG;
-//    using MongoDB.Driver.Linq;
-//    using System.ComponentModel.DataAnnotations;
-//    using ZstdSharp.Unsafe;
-
-//    [DataStreamUserRespresentation("VWAP")]
-//    public class VWAPStream : ViewStream {
-//       // returns the VWAP stream for each contract 
-//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-//          // stream is grouped per contract and then vwap is calculated per contract
-//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-//          (rows, idx) => {
-//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-//             IBlueRow ir = rows.ElementAt((int)idx);
-//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-//             //vwap = -313.987;
-//             nr.setValue("vwap", vwap);
-//             return nr;
-//          });
-//          return rs;
-//       }
-//       // the args want dt1 > dt2, if its nto we, swap them
-//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-//          if (dt1 < dt2) {
-//             DateTime aux = dt1;
-//             dt1 = dt2;
-//             dt2 = aux;
-//          }
-//       }
-
-//       // this assumes the ohlc_str has diff contracts
-//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-//          swap_if_necessary(ref begin_dt, ref end_dt);
-//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-//          string symbol = String.Empty;
-//          symbol = ccat_sym_contr.Item2;
-//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-//          ViewStream rs = vwap_str;
-//          switch (ccat_sym_contr.Item1) {
-//             case ContractUtils.ContractCategory.FutureString:
-//             case ContractUtils.ContractCategory.SpreadString:
-//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-//                break;
-//             case ContractUtils.ContractCategory.Symbol:
-//                rs = new FuturesActiveContractStream(vwap_str);
-//                break;
-//          }
-//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-//          rs = new NewProjectStream(rs, "! vwap");
-//          mySetRealSource(rs);
-//       }
-
-//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-//       :
-//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-//    }
-// }
-
-using System;
-
-using BCG;
-
-using bts.api;
-
 namespace bts.SilverPine {
-   /*
-      ----------------------------------------------------------------------
-                                  *Expression*
-      ----------------------------------------------------------------------
-          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
-
-          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-          r=math(r, "vprice = volume * close")
-          r=aggregate(r, "vprice:Sum, volume:Sum")
-          r=math(r, "vwap = vprice_sum / volume_sum")
-          r=newProject(r, "vwap !")
-
-      ----------------------------------------------------------------------
-                                    *Schema*
-      ----------------------------------------------------------------------
-          id                              System.Int32
-          vwap                            System.Double
-   */
+   using bts;
+   using BCG;
+   using MongoDB.Driver.Linq;
+   using System.ComponentModel.DataAnnotations;
+   using ZstdSharp.Unsafe;
 
    [DataStreamUserRespresentation("VWAP")]
    public class VWAPStream : ViewStream {
-      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-      :
-      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
-
-      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-         var fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
-
-         r = new MathStream(r,
-                 "vprice = volume * close");
-         r = new NonIncGroupByStream(r,
-                                      "vprice:Sum, volume:Sum");
-         r = new MathStream(r,
-                             "vwap = vprice_sum / volume_sum");
-         r = new NewProjectStream(r,
-                                   "vwap !");
-         mySetRealSource(r);
+      // returns the VWAP stream for each contract 
+      private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+         ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+         // stream is grouped per contract and then vwap is calculated per contract
+         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+         (rows, idx) => {
+            Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+            IBlueRow ir = rows.ElementAt((int)idx);
+            var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+            var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+            double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+            MutableBlueRow nr = new MutableBlueRow(schema, ir);
+            //vwap = -313.987;
+            nr.setValue("vwap", vwap);
+            return nr;
+         });
+         return rs;
       }
+      // the args want dt1 > dt2, if its nto we, swap them
+      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+         if (dt1 < dt2) {
+            DateTime aux = dt1;
+            dt1 = dt2;
+            dt2 = aux;
+         }
+      }
+
+      // this assumes the ohlc_str has diff contracts
+      public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+         swap_if_necessary(ref begin_dt, ref end_dt);
+         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+         string symbol = String.Empty;
+         symbol = ccat_sym_contr.Item2;
+         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+         ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+         // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+         // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+         ViewStream rs = vwap_str;
+         switch (ccat_sym_contr.Item1) {
+            case ContractUtils.ContractCategory.FutureString:
+            case ContractUtils.ContractCategory.SpreadString:
+               rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+               break;
+            case ContractUtils.ContractCategory.Symbol:
+               rs = new FuturesActiveContractStream(vwap_str);
+               break;
+         }
+         rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+         rs = new NewProjectStream(rs, "! vwap");
+         mySetRealSource(rs);
+      }
+
+      public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+      :
+      this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -75,36 +75,4 @@ namespace bts.SilverPine {
          mySetRealSource(rs);
       }
    }
-}
-
-
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         // filtering for futures only
-         r = new FilterStream(r, r => {
-            string contract = r.get<string>("symbol");
-            var fc = new ContractUtils.FutureContract(contract);
-            return fc.future_tradable_contract != null;
-         });
-         r = new NonIncGroupByStream(r,
-                                      "ts_event",
-                                      "volume:Max");
-         r = new NewProjectStream(r,
-                                   "ts_event symbol !");
-         r = new SortStream(r,
-                             "-ts_event");
-         mySetRealSource(r);
-      }
-   }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -2,93 +2,6 @@ namespace bts.SilverPine {
    using bts;
    using BCG;
    using MongoDB.Driver;
-
-   // OHLCV request breakdown
-   // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it)
-   // bar_size (one min, one day)
-   // live/historic
-   // time range
-
-   // internal code oranization only
-   public class OHLCV_1min_stream : ViewStream {
-
-      public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) :
-      this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { }
-
-      public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) {
-         Func<string, bool, SQLiteStream> get_sqlite_str = (symbol, is_continous_contracts) => {
-            string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min";
-            string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3";
-            string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
-               range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss");
-            string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
-            return new SQLiteStream(db_str, sql_str);
-         };
-
-         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
-         var contract_cat = tup.Item1;
-         switch (contract_cat) {
-            case ContractUtils.ContractCategory.Symbol: {
-                  ViewStream res = get_sqlite_str(tup.Item2, true);
-                  mySetRealSource(res);
-                  // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it
-                  // ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false);
-                  // // the following is whats written to the DB in the _contionous table
-                  // ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event");
-                  // ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2);
-                  // ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume",
-                  //                                     active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract");
-                  // res = new NullFilterStream(res, "continous_contract", true);
-                  // res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !");
-                  // mySetRealSource(res);
-                  break;
-               }
-            default: {
-                  ViewStream sql_str = get_sqlite_str(tup.Item2, false);
-                  ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'");
-                  // filter here
-                  mySetRealSource(rs);
-                  break;
-               }
-         }
-      }
-   }
-
-   // this is to make bigger bars form smaller bars. Right now the assumption it only works
-   // with symbols vs contracts, and the 1min <= aggregation_period < 1day
-   [DataStreamUserRespresentation("OHLCVAggregated")]
-   public class OHLCVAggregatedStream : ViewStream {
-      public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) :
-      this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) {
-      }
-
-      public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) {
-         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol);
-         Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol);
-         if (aggregation_period < TimeSpan.FromDays(1.0) &&
-            aggregation_period > TimeSpan.FromMinutes(1.0)) {
-            ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end);
-            r = new DateTimeExpanstionStream(r,
-                                              "ts_event");
-            double tot_secs = aggregation_period.TotalSeconds;
-            r = new MathStream(r,
-                                $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})");
-            r = new NonIncGroupByStream(r,
-                                         "bucket",
-                                         // column:operation
-                                         "open:first, high:max, low:min, close:last, volume:sum",
-                                         "ts_event");
-            r = new SortStream(r,
-                                "-ts_event");
-            mySetRealSource(r);
-         } else {
-            // fix it later
-            Log.assert(false);
-         }
-      }
-   }
-
-
    [DataStreamUserRespresentation("OHLCV")]
    // This stream retreives the OHLCV table as is, and fires it as-is. 
    // The only logic here is as follows:
@@ -167,13 +80,12 @@ namespace bts.SilverPine {
          switch (ohlcv_params.type) {
             case OHLCVType.LIVE:
             case OHLCVType.LIVE_AND_HISTORIC:
-               // TODO
                Log.assert(false);
                break;
             case OHLCVType.HISTORIC:
                // TODO: assuming 1 day only for historic               
                string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
-                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
+               //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
                string db_str = $"Data Source={_location}{_db_name};Version=3";
                string sql_str = $"select * from {table_name}";
                // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
@@ -199,12 +111,14 @@ namespace bts.SilverPine {
       :
       this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }
 
-      // aggregates the bars
-      public OHLCVStream(string symbol, TimeSpan agregation_period) {
-
+      internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) {
+         ViewStream rs = source;
+         rs = new SchemaGuesserStream(rs);
+         if (bar_size == OHLCVBarSize.ONE_DAY) {
+            rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+         }
+         rs = new NewProjectStream(rs, "symbol:contract");
+         mySetRealSource(rs);
       }
-
-
-
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -3,22 +3,49 @@ namespace bts.SilverPine {
 
    using bts;
 
+   using DnsClient.Protocol;
+
+   using MongoDB.Driver;
+
    [DataStreamUserRespresentation("Settlement")]
    public class SettlementPriceStream : ViewStream {
 
       public SettlementPriceStream(string future_name, string date_txt)
-      : this(future_name, date_txt, date_txt) { }
+      : this(future_name, date_txt, String.Empty) { }
 
       public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
       : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
       public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
-      : this(symbol_or_contract, begin_dt, begin_dt) { }
+      : this(symbol_or_contract, begin_dt, DateTime.MinValue) { }
 
       public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
-         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
+         // DateTime begin_dt = DateTime.Parse(begin_dt_txt);
+         // DateTime end_dt = String.IsNullOrEmpty(end_dt_txt) ? DateTime.MinValue : DateTime.Parse(end_dt_txt);
+         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+         ContractUtils.ContractCategory contr_cat = ccat_sym_contr.Item1;
+         string symbol = ccat_sym_contr.Item2;
+         ViewStream settl_str = new OHLCVStream(symbol);
+         // TODO: close != settlement, get settlement from somewhere
+         ViewStream rs = new NewProjectStream(settl_str, "close:settlement");
+         switch (contr_cat) {
+            case ContractUtils.ContractCategory.FutureString:
+            case ContractUtils.ContractCategory.SpreadString:
+               rs = new FilterStream(settl_str, r => r.get<string>("contract").CompareTo(symbol_or_contract) == 0);
+               break;
+            case ContractUtils.ContractCategory.Symbol:
+               rs = new FuturesActiveContractStream(settl_str);
+               break;
+         }
+         if (DateTime.MinValue.Equals(end_dt)) {
+            rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+         }
+         rs = new FilterStream(rs, r => {
+            DateTime dt = r.get_datetime("ts_event");
+            return dt <= begin_dt && dt >= end_dt;
+         });
+         rs = new MathStream(rs, "settlement = close * 1.0");
+         rs = new NewProjectStream(rs, " ! ts_event:date settlement");
          mySetRealSource(rs);
       }
 

src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json

@@ -41,74 +41,70 @@
       },
       {
          "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
          "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
          "examples": [
             {
                "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
-               "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
+               "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
             },
             {
                "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
+               "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
             },
             {
                "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
+               "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
             },
             {
                "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
-               "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
+               "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
             }
          ]
       },
       {
-         "category": "FuturePriceDiff",
-         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
-         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
+         "category": "Price Diff",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
+         "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
          "examples": [
             {
                "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
-               "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
+               "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
             },
             {
                "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
+               "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
             },
             {
                "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
-               "dsl": "FuturePriceDiff(\"NGZ4\", 8/20/2024, 8/30/2024)"
-            },
-            {
-               "natural_language": "What is the price move in  SP 500 Future from Aug 30th 2023 to October 10th 2023",
-               "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
-            },
-            {
-               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
-               "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
+               "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
             }
          ]
       },
       {
-         "category": "Aggregate Volume",
-         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
+         "category": "Absolute Volume",
+         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
          "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
          "examples": [
             {
-               "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
-               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
+               "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
+               "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
             },
             {
-               "natural_language": "Which Gold Spread traded the most in the first month of this year",
-               "dsl": "VolumeAgg(\"GC\", \"FutureSpread\", 1/1/2024, 1/31/2024)"
+               "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
+               "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
             },
             {
-               "natural_language": "Which 10 year Spread traded the highest volume last week to now?(in this example(only) we are assuming today is 11/26/2024)",
-               "dsl": "VolumeAgg(\"ZN\", \"FutureSpread\", 11/19/2024, 11/26/2024)"
+               "natural_language": "Which 10 year Spread traded the highest volume last week?",
+               "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
             },
             {
-               "natural_language": "Which Corn Spread traded the most today (in this example(only) we are assuming today is 11/26/2024)?",
-               "dsl": "VolumeAgg(\"ZC\", \"FutureSpread\", 11/26/2024, 11/26/2024)"
+               "natural_language": "Which Corn Spread traded the most today?",
+               "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
+            },
+            {
+               "natural_language": "Which Gold Future traded the most today?",
+               "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
             }
          ]
       },
@@ -222,17 +218,6 @@
                "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
             }
          ]
-      },
-      {
-         "category": "Price Movement",
-         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking for the average price movement for e-minis (aka ES aka SP Futures aka SP 500 Futures)) on Wednesday between 9:30 am and 9:40 AM?  the multiline DSL is as follows (along with the explaination in comments before each line, dont include the comments in the asnwer):  //following line gets the 1 min ohlcv bars of ES \n r=OHLCV_1min_stream(\"ES\")\n //this passes it to the filter stream, which filters the rows for Wednesday, and time bw 9:30 and 9:40.\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\n// this aggregates it using the date as the agregation key, it find the max of the column called high, and the low of the column called min, to find the max and the min prices in that time window\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\n/// this then passes the previous result to the math stream operator which calculates the move, which is the difference between high_max and low_min (columns that were added from the previous aggregate)\nr=math(r,\"move = high_max - low_min\")\n// At the end we compute the mean of the move column, notice this is the total mean, so we are not providing an aggregation col to agregate\nr=aggregate(r, \"move:mean\")\n// we project out every col by the ! operator and call rename move_mean as average_vol\nr=newProject(r,\"move_mean:average_move !\")",
-         "prompt": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM ?",
-         "examples": [
-            {
-               "natural_language": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM?",
-               "dsl": "r=OHLCV_1min_stream(\"ES\")\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\nr=math(r,\"move = high_max - low_min\")\nr=aggregate(r, \"move:mean\")\nr=newProject(r,\"move_mean:average_move !\")"
-            }
-         ]
       }
    ]
 }

src/app_sparkd/sparkd/cmds/Run.cpp

@@ -12,6 +12,7 @@
 #include "spark/md/service/MarketDataServiceServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"
+#include "spark/orderman/OrderServiceServer.h"
 #include "spark/orderman/events/TradeSummarySignalDispatcher.h"
 #include "spark/orderman/order_factory/DynamicOrderRegistry.h"
 #include "spark/persistence/IDGenerator.h"
@@ -24,7 +25,6 @@
 #include "sparkd/algo/AlgoDefinitionServiceServer.h"
 #include "sparkd/analytics/AnalyticsServiceServer.h"
 #include "sparkd/cmds/md/cmds/Cleanup.h"
-#include "sparkd/execution/OrderServiceServer.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
 #include "sparkd/md/MarketDataGateway.h"

src/client/spark/SpreadBox/DateTimeFormatter.cs

@@ -0,0 +1,41 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+
+// namespace BCG
+// {
+//    public class DateTimeFormatter
+//    {
+//       private delegate string MyFormatDateTime (DateTime v);
+//       public enum ExpirationShortForms
+//       {
+//          DayMonthYearDiff2000,
+//          MonthYearDiff2000
+//       }
+//       public enum DateTimeFormats { Regular, ShortDate, DayMonthYearDiff2000, MonthYearDiff2000 }
+//       private static Dictionary<string, MyFormatDateTime> _local_formats = null;
+//       private static void _init()
+//       {
+//          if (_local_formats == null)
+//          {
+//             _local_formats = new Dictionary<string, MyFormatDateTime>();
+//             _local_formats.Add(DateTimeFormats.Regular.ToString(), v => v.ToString());
+//             _local_formats.Add(DateTimeFormats.ShortDate.ToString(), v => v.ToShortDateString());
+//             _local_formats.Add(DateTimeFormats.DayMonthYearDiff2000.ToString(),
+//                (DateTime v) => { return String.Format("{0} {1}{2}", v.Day, BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); });
+//             _local_formats.Add(DateTimeFormats.MonthYearDiff2000.ToString(),
+//                (DateTime v) => { return String.Format("{0}{1}", BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); });            
+//          }
+//       }
+//       public static string format(DateTime val, string format)
+//       {
+//          _init ();
+//          MyFormatDateTime f = null;
+//          if (_local_formats.TryGetValue(format, out f))
+//             return f(val);
+//          else
+//             return val.ToString(format);
+//       }
+//    }
+// }

src/client/spark_api_server/Configurations/spark_api_server.json

@@ -1,61 +0,0 @@
-{
-  "user": "bts",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://localhost:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://localhost:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
-    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
-  }
-}

src/client/spark_api_server/Program.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.websocket_core;
 
 using Bts.DataStreamCore.Services;

src/client/spark_api_server/RunOptions.cs

@@ -1,13 +1,6 @@
 namespace Spark.Api.Server {
-   using Bts.PubSub;
-
    using CommandLine;
 
-   using System;
-   using System.IO;
-   using System.Text.Json;
-   using System.Text.Json.Serialization;
-
    [Verb("run")]
    public class RunOptions {
       [Option("user", Default = CommonOptionMetadata.DEFAULT_USER)]
@@ -21,44 +14,5 @@ namespace Spark.Api.Server {
          MetaValue = CommonOptionMetadata.SPARKD_META_VALUE,
          HelpText = CommonOptionMetadata.SPARKD_HELP_TEXT)]
       public string? sparkd { get; set; }
-
-      [JsonPropertyName("pubsub")]
-      public PubSubConfig? PubSub { get; set; }
-
-      [JsonIgnore]
-      [Option(
-         "config",
-         MetaValue = "PATH",
-         HelpText = "Path to json file representing these options (RunOptions)")]
-      public string? ConfigFile { get; set; }
-
-      public static RunOptions Default() =>
-         new() {
-            PubSub = PubSubConfig.Default(),
-         };
-
-      public void MergeWith(RunOptions other) {
-         sparkd ??= other.sparkd;
-         PubSub ??= other.PubSub;
-         ConfigFile ??= other.ConfigFile;
-      }
-
-      /// <summary>
-      /// Merges options with those defined in `ConfigFile` (does not overwrite
-      /// non-null existing values).
-      /// </summary>
-      /// <exception cref="InvalidOperationException">
-      /// JsonSerializer.Deserialize returns null
-      /// </exception>
-      public void MergeWithConfigFile() {
-         if (ConfigFile is null) {
-            return;
-         }
-         var cfgString = File.ReadAllText(ConfigFile);
-         var parsedCfg = JsonSerializer.Deserialize<RunOptions>(cfgString) ??
-            throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
-         MergeWith(parsedCfg);
-         ConfigFile = null;
-      }
    }
 }

src/client/spark_api_server/appsettings.Staging.json

@@ -1 +1,5 @@
-{}
+{
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  }
+}

src/client/spark_api_server/appsettings.json

@@ -20,9 +20,6 @@
     "KeepAliveInterval": "00:02:00",
     "ReceiveBufferSize": 4096
   },
-  "Spark": {
-    "ConfigServer": "http://sparkd:8000/config"
-  },
   "SparkApi": {
     "Credentials": {
       "User": "bts",

src/lib_http/CMakeLists.txt

@@ -26,7 +26,3 @@ target_link_libraries(${PROJECT_NAME} PUBLIC unofficial::curlpp::curlpp)
 # concurrencpp
 find_package(concurrencpp CONFIG REQUIRED)
 target_link_libraries(${PROJECT_NAME} PUBLIC concurrencpp::concurrencpp)
-
-# websocketpp
-find_package(websocketpp CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} PUBLIC websocketpp::websocketpp)

src/lib_http/http/http_authorization.h

@@ -1,78 +0,0 @@
-#pragma once
-
-#include "http/http_client.h"
-#include "system/time_stamp.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::http {
-
-/**
- * Provides a convenient abstraction for http (o)authentication. Oauth requests
- * are sent lazily, when `get_token` is called, so the caller doesn't need
- * to think about it. Responses are stored as member variables, so subsequent
- * calls to `resolve_token()` don't request another token until the current one
- * has expired.
- */
-class HttpAuthorization {
- public:
-   HttpAuthorization() = default;
-   HttpAuthorization(
-       std::string url, std::string client_id, std::string client_secret);
-
-   HttpAuthorization(const HttpAuthorization &) = default;
-   HttpAuthorization(HttpAuthorization &&) = default;
-   HttpAuthorization &operator=(const HttpAuthorization &) = default;
-   HttpAuthorization &operator=(HttpAuthorization &&) = default;
-
-   /**
-    * Gets the current access token, or requests a new one if it is expired,
-    * or \p force_new_request is true
-    *
-    * @param force_new_request Request a new token even if the current one hasn't expired
-    */
-   std::string &get_token(bool force_new_request = false);
-
-   /**
-    * Gets a boolean indicating whether or not the current access token is expired.
-    */
-   bool is_expired() const;
-
- private:
-   std::string url;
-   std::string client_key;
-   std::string client_secret;
-
-   bts::core::TimeStamp exp_time;
-   std::string access_token;
-   std::string token_type;
-};
-
-void to_json(nlohmann::json &j, const HttpAuthorization &c);
-void from_json(const nlohmann::json &j, HttpAuthorization &c);
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline HttpAuthorization::HttpAuthorization(
-    std::string url, std::string client_id, std::string client_secret)
-    : url(std::move(url)), client_key(std::move(client_id)),
-      client_secret(std::move(client_secret)) {}
-
-inline std::string &HttpAuthorization::get_token(bool force_new_request) {
-   if (!force_new_request && !is_expired()) {
-      return access_token;
-   }
-   std::int64_t exp_secs;
-   std::tie(access_token, token_type, exp_secs) =
-       HttpClient::get_oauth_token(url, client_key, client_secret);
-   exp_time = core::TimeStamp::now() + core::TimeSpan::from_seconds(exp_secs);
-   return access_token;
-}
-
-inline bool HttpAuthorization::is_expired() const {
-   return exp_time <= core::TimeStamp::now();
-}
-
-} // namespace bts::http

src/lib_http/http/http_client.cpp

@@ -80,19 +80,6 @@ void HttpClient::post(
    perform(curl_req, response);
 }
 
-void HttpClient::put(
-    const string &url,
-    const string &body,
-    const unordered_map<string, string> &headers,
-    IHttpResponse *response) {
-   _LOG_REQ_BEGIN(PUT, url, headers);
-   curlpp::Easy curl_req;
-   init_request_common(curl_req, url, headers);
-   curl_req.setOpt<curlpp::options::Put>(true);
-   curl_req.setOpt<curlpp::options::PostFields>(body);
-   perform(curl_req, response);
-}
-
 void HttpClient::get(
     const string &url,
     const unordered_map<string, string> &headers,

src/lib_http/http/http_client.h

@@ -47,15 +47,6 @@ class HttpClient {
        const std::unordered_map<std::string, std::string> &headers,
        IHttpResponse *response);
 
-   /**
-    * Performs an http PUT request
-    */
-   static void put(
-      const std::string &url,
-      const std::string &body,
-       const std::unordered_map<std::string, std::string> &headers,
-       IHttpResponse *response);
-
    /**
     * Requests an oauth access token
     *

src/lib_http/http/http_method.h

@@ -13,8 +13,7 @@ namespace bts::http {
 #define FOR_ALL_VALUES(per_value)                                    \
    per_value(HttpMethod, NONE)                                       \
    per_value(HttpMethod, GET)                                        \
-   per_value(HttpMethod, POST)                                       \
-   per_value(HttpMethod, PUT)
+   per_value(HttpMethod, POST)
 // clang-format on
 
 /**

src/lib_http/http/websocket/websocket_client.cpp

@@ -1,379 +0,0 @@
-#include "websocket_client.h"
-
-#include "http/http_authorization.h"
-#include "http/websocket/websocket_client_state.h"
-#include "http/websocket/websocket_close_status.h"
-#include "http/websocket/websocket_error_code.h"
-#include "http/websocket/websocketpp_exception_formatter.h"
-#include "system/facades/bts_assert.h"
-#include "system/formatters/exception_formatter.h"
-#include "system/lock_guard.h"
-#include "system/logging/logging.h"
-#include "system/time_span.h"
-#include "system/time_stamp.h"
-
-#include <atomic>
-#include <exception>
-#include <fmt/format.h>
-#include <nlohmann/json.hpp>
-#include <optional>
-#include <stdexcept>
-#include <string>
-#include <thread>
-#include <websocketpp/close.hpp>
-#include <websocketpp/common/connection_hdl.hpp>
-#include <websocketpp/common/memory.hpp>
-#include <websocketpp/common/system_error.hpp>
-#include <websocketpp/common/thread.hpp>
-#include <websocketpp/config/asio_client.hpp>
-#include <websocketpp/config/core_client.hpp>
-#include <websocketpp/roles/client_endpoint.hpp>
-
-namespace bts::websocket {
-
-struct WebsocketClientSnapshot {
-   int client_id;
-   WebsocketClientState state;
-   int version;
-};
-
-} // namespace bts::websocket
-
-FMT_IMPL(
-    bts::websocket::WebsocketClientSnapshot,
-    t,
-    "id: {}; state: {}; version: {}",
-    t.client_id,
-    t.state,
-    t.version);
-
-namespace bts::websocket {
-
-using namespace std::chrono_literals;
-
-using bts::core::TimeSpan;
-using bts::core::TimeStamp;
-using nlohmann::json;
-using std::map;
-using std::optional;
-using std::string;
-using websocketpp::config::asio_tls_client;
-
-using WebsocketppClient =
-    websocketpp::client<websocketpp::config::asio_tls_client>;
-using ConnectionHdl = websocketpp::connection_hdl;
-using ConnectionPtr = WebsocketppClient::connection_ptr;
-using MessageTypePtr = websocketpp::config::core_client::message_type::ptr;
-using SslContext = boost::asio::ssl::context;
-using State = WebsocketClientState;
-
-// ws endpoint http://markets.api.uat.cmegroup.com/marketdatastream/v1
-
-#define _ASSERT_MSGF(condition, ...)                                           \
-   bts_assert_msgf(condition, "WebsocketClient::" __VA_ARGS__)
-
-#define DIAG(...) LOG(Gen, Diag, __VA_ARGS__)
-#define DIAG_FN(...) LOG(Gen, Diag, "WebsocketClient::" __VA_ARGS__)
-#define INFO(...) LOG(Gen, Info, __VA_ARGS__)
-#define INFO_FN(...) LOG(Gen, Info, "WebsocketClient::" __VA_ARGS__)
-#define WARN(...) LOG(Gen, Warn, __VA_ARGS__)
-#define WARN_FN(...) LOG(Gen, Warn, "WebsocketClient::" __VA_ARGS__)
-#define ERROR(...) LOG(Gen, Error, __VA_ARGS__)
-#define ERROR_FN(...) LOG(Gen, Error, "WebsocketClient::" __VA_ARGS__)
-
-struct WebsocketClient::PImpl {
-   std::string url;
-   WebsocketppClient client;
-   std::uint32_t timeout_secs;
-   WebsocketppClient::connection_ptr con;
-   std::optional<MessageHandler> message_handler;
-   std::size_t sub_id_counter{0};
-   string server_name;
-   std::optional<std::thread> io_thread;
-   int client_id{client_id_counter++};
-   std::mutex instance_mx;
-   State state;
-   int version = 0;
-
-   ~PImpl() {
-      if (io_thread) {
-         io_thread->join();
-      }
-   }
-
-   int set_state(State state) {
-      if (this->state == state) {
-         return version;
-      }
-      version++;
-      INFO(
-          "State changing from {} -> {}. id: {}; version: {}", this->state,
-          state, client_id, version);
-      this->state = state;
-      return version;
-   }
-
-   constexpr WebsocketClientSnapshot snapshot() const {
-      return {client_id, state, version};
-   }
-
-   void close(websocketpp::close::status::value code, std::string reason) {
-      BTS_LOCK_GUARD(instance_mx);
-      INFO_FN(
-          "close. {}; code: {}; reason: {}", snapshot(),
-          WebsocketCloseStatus{code}, reason);
-      switch (state.value) {
-      case State::OPENING:
-         WARN_FN(
-             "close. Cannot close while websocket is still opening. {}",
-             snapshot());
-         // Wait until we get the 'open' callback, then initiate the close
-         set_state(State::WAITING_TO_CLOSE);
-         return;
-      case State::OPEN:
-         set_state(State::CLOSING);
-         break;
-      default:
-         return;
-      }
-      websocketpp::lib::error_code ec;
-      client.close(con->get_handle(), code, reason, ec);
-      if (ec) {
-         ERROR_FN(
-             "close. {}; code: {}; reason: {}; error: {}", snapshot(),
-             WebsocketCloseStatus{code}, reason, ec);
-      }
-   }
-
-   websocketpp::lib::error_code send_text(const string &text) {
-      INFO_FN("send_text. {}; text: {}", snapshot(), text);
-      websocketpp::lib::error_code ec;
-      client.send(con, text, websocketpp::frame::opcode::value::TEXT, ec);
-      if (ec) {
-         ERROR_FN("send_text. {}; text: {}; err: {}", snapshot(), text, ec);
-      }
-      return ec;
-   }
-
-   /**
-    * Connects to the websocket server
-    */
-   void connect(const map<string, string> &headers) {
-      connect_in_background(headers);
-      // Wait for connect
-      if (timeout_secs > 0) {
-         wait_for_connect(TimeSpan::from_seconds(timeout_secs));
-      }
-   }
-
-   /**
-    * Connects to the websocket server in the background.
-    *
-    * @note Doesn't block while waiting for the connection to be established
-    */
-   void connect_in_background(const map<string, string> &headers) {
-      BTS_LOCK_GUARD(instance_mx);
-      INFO_FN("connect_no_timeout. {}", snapshot());
-      switch (state.value) {
-      case State::IDLE:
-         break;
-      default:
-         WARN_FN("connect_no_timeout. Unexpected state. {}", snapshot());
-         return;
-      }
-      set_state(State::OPENING);
-      try {
-         init_logging();
-         init_handlers();
-         // Get connection
-         websocketpp::lib::error_code ec;
-         con = client.get_connection(url, ec);
-         if (ec) {
-            ERROR_FN("connect_no_timeout. {}; err: {}", snapshot(), ec);
-            return;
-         }
-         // Add auth header (if non-empty)
-         for (const auto &[k, v] : headers) {
-            con->append_header(k, v);
-         }
-         // Connect
-         client.connect(con);
-         bts_assert_msgf(
-             !io_thread, "Websocket io_thread already instantiated. {}",
-             snapshot());
-         io_thread = std::thread{[this]() {
-            INFO("Websocket io thread begin. {}", snapshot());
-            try {
-               client.run();
-            } catch (const websocketpp::exception &ex) {
-               ERROR_FN(
-                   "io_thread. Websocket io threw websocketpp::exception. {}; "
-                   "err: {}",
-                   snapshot(), ex);
-            } catch (const std::exception &ex) {
-               ERROR_FN(
-                   "io_thread. Websocket io threw std::exception. {}; err: {}",
-                   snapshot(), ex);
-            } catch (...) {
-               bts_assert_msgf(
-                   false,
-                   "io_thread. Websocket io threw unknown exception. Could "
-                   "this have been thrown by a callback delegate? {}",
-                   snapshot());
-            }
-            INFO("Websocket 'run' thread end. {}", snapshot());
-         }};
-      } catch (const std::exception &ex) {
-         ERROR_FN(
-             "connect_no_timeout. Failed to connect. {}; err: {}", snapshot(),
-             ex);
-         set_state(State::IDLE);
-         // Don't wait for connect
-         return;
-      }
-   }
-
-   void wait_for_connect(const TimeSpan timeout) {
-      INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout);
-      const auto start = TimeStamp::now();
-      while (state == State::OPENING) {
-         const auto elapsed = TimeStamp::now() - start;
-         const auto rem = timeout - elapsed;
-         if (rem < TimeSpan::zero()) {
-            throw std::runtime_error{fmt::format(
-                "Timed out waiting for connection after {} secs. {}",
-                elapsed.total_seconds(), snapshot())};
-         }
-         WARN_FN(
-             "wait_for_connect. Still waiting. {}; timeout: {}; rem: {}",
-             snapshot(), timeout, rem);
-         std::this_thread::sleep_for(std::chrono::seconds(1));
-      }
-   }
-
- private:
-   static std::atomic<int> client_id_counter;
-
-   auto on_tls_init(ConnectionHdl /*hdl*/) {
-      INFO_FN("on_tls_init. state: {}; version: {}", state, version);
-      auto result =
-          websocketpp::lib::make_shared<SslContext>(SslContext::sslv23_client);
-      result->set_verify_mode(boost::asio::ssl::verify_none);
-      return result;
-   }
-
-   void init_handlers() {
-      INFO_FN("init_handlers. state: {}; version: {}", state, version);
-      bts_assert(!io_thread);
-      client.init_asio();
-      client.set_message_handler(
-          [this](auto hdl, auto msg) { on_message(hdl, msg); });
-      client.set_open_handler([this](auto hdl) { on_open(hdl); });
-      client.set_tls_init_handler(
-          [this](auto hdl) { return on_tls_init(hdl); });
-      client.set_close_handler([this](auto hdl) { on_close(hdl); });
-      client.set_fail_handler([this](auto hdl) {
-         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
-         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
-      });
-   }
-
-   void init_logging() {
-      INFO_FN("init_logging. {}", snapshot());
-      client.set_access_channels(websocketpp::log::alevel::all);
-      client.clear_access_channels(websocketpp::log::alevel::frame_payload);
-      client.set_error_channels(websocketpp::log::elevel::all);
-   }
-
-   void on_open(ConnectionHdl hdl) {
-      BTS_LOCK_GUARD(instance_mx);
-      INFO_FN("on_open. {}", snapshot());
-      const auto ec = client.get_con_from_hdl(hdl)->get_ec();
-      if (ec) {
-         ERROR_FN("on_open. {}; err: {}", snapshot(), ec.message());
-      }
-      switch (state.value) {
-      case State::OPENING:
-         break;
-      default:
-         WARN_FN("on_open. Unexpected state. {}", snapshot());
-         return;
-      }
-      set_state(State::OPEN);
-   }
-
-   void on_message(ConnectionHdl /*hdl*/, MessageTypePtr msg) {
-      DIAG("on_message. {}", snapshot());
-      const auto &payload = msg->get_payload();
-      if (message_handler) {
-         (*message_handler)(payload);
-         return;
-      }
-      ERROR_FN(
-          "on_message. Message handler is null. {}; payload: \n{}", snapshot(),
-          payload);
-   }
-
-   void on_close(ConnectionHdl hdl) {
-      BTS_LOCK_GUARD(instance_mx);
-      INFO_FN("on_close. {}", snapshot());
-      switch (state.value) {
-      case State::CLOSING:
-         break;
-      default:
-         WARN_FN("on_close. Unexpected state. {}", snapshot());
-         break;
-      }
-      set_state(State::IDLE);
-      const auto con = client.get_con_from_hdl(hdl);
-      const auto ec = con->get_ec();
-      if (ec) {
-         ERROR_FN(
-             "on_close. {}; local_code: {}; "
-             "local_reason: {}; remote_code: {}; remote_reason: {}; err: {}",
-             snapshot(), con->get_local_close_code(),
-             con->get_local_close_reason(), con->get_remote_close_code(),
-             con->get_remote_close_reason(), ec.message());
-      }
-   }
-};
-
-WebsocketClient::WebsocketClient(std::string url, std::uint32_t timeout_secs)
-    : me_(std::make_unique<PImpl>()) {
-   me_->url = std::move(url);
-   me_->timeout_secs = timeout_secs;
-}
-
-WebsocketClient::~WebsocketClient() {
-   me_->close(websocketpp::close::status::going_away, "");
-}
-
-std::atomic<int> WebsocketClient::PImpl::client_id_counter = 0;
-
-WebsocketClient::MessageHandler &WebsocketClient::message_handler() {
-   if (!me_->message_handler) {
-      me_->message_handler.emplace();
-   }
-   return *me_->message_handler;
-}
-
-void WebsocketClient::connect(const map<string, string> &headers) {
-   me_->connect(headers);
-}
-
-websocketpp::lib::error_code WebsocketClient::send_text(const string &text) {
-   return me_->send_text(text);
-}
-
-void WebsocketClient::disconnect() {
-   me_->close(
-       websocketpp::close::status::going_away,
-       "WebsocketClient::disconnect was called");
-}
-
-bool WebsocketClient::is_connected() const {
-   BTS_LOCK_GUARD(me_->instance_mx);
-   return me_->state == State::OPEN;
-}
-
-} // namespace bts::websocket

src/lib_http/http/websocket/websocket_client.h

@@ -1,37 +0,0 @@
-#pragma once
-
-#include "http/http_authorization.h"
-#include "system/delegate.h"
-
-#include <memory>
-#include <string>
-#include <websocketpp/config/core_client.hpp>
-#include <websocketpp/message_buffer/message.hpp>
-
-namespace bts::http {
-class HttpAuthorization;
-}
-
-namespace bts::websocket {
-
-class WebsocketClientConfig;
-
-class WebsocketClient {
- public:
-   WebsocketClient(std::string url, std::uint32_t timeout_secs = 20);
-   ~WebsocketClient();
-
-   using MessageHandler = bts::core::Delegate<void(const std::string &)>;
-
-   void connect(const std::map<std::string, std::string> &headers = {});
-   void disconnect();
-   bool is_connected() const;
-   MessageHandler &message_handler();
-   websocketpp::lib::error_code send_text(const std::string &text);
-
- private:
-   struct PImpl;
-   std::unique_ptr<PImpl> me_;
-};
-
-} // namespace bts::websocket

src/lib_http/http/websocket/websocket_client_config.h

@@ -1,39 +0,0 @@
-#pragma once
-
-#include "system/enum_macros.h"
-#include "system/json/json_macros.h"
-
-#include <nlohmann/json.hpp>
-#include <string>
-
-namespace bts::websocket {
-
-class WebsocketClientConfig {
- public:
-   std::string url;
-   std::string oauth_url;
-   std::string client_id;
-   std::string client_secret;
-   std::uint32_t timeout_secs;
-};
-
-inline void to_json(nlohmann ::json &j, const WebsocketClientConfig &src) {
-   j["url"] = src.url;
-   j["oauth_url"] = src.oauth_url;
-   j["client_id"] = src.client_id;
-   j["client_secret"] = src.client_secret;
-   j["timeout_secs"] = src.timeout_secs;
-}
-
-inline void from_json(const nlohmann::json &j, WebsocketClientConfig &cfg) {
-   j.at("url").get_to(cfg.url);
-   j.at("oauth_url").get_to(cfg.oauth_url);
-   j.at("client_id").get_to(cfg.client_id);
-   j.at("client_secret").get_to(cfg.client_secret);
-   const auto timeout_secs = j.find("timeout_secs");
-   if (timeout_secs != j.end()) {
-      timeout_secs->get_to(cfg.timeout_secs);
-   }
-}
-
-} // namespace bts::websocket

src/lib_http/http/websocket/websocket_client_state.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include "system/FMT_IMPL.h"
-#include "system/enum_macros.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::websocket {
-
-// clang-format off
-#define FOR_ALL_VALUES(op)                    \
-  op(WebsocketClientState, IDLE)              \
-  op(WebsocketClientState, OPENING)           \
-  op(WebsocketClientState, OPEN)              \
-  op(WebsocketClientState, WAITING_TO_CLOSE)  \
-  op(WebsocketClientState, CLOSING)
-// clang-format on
-
-ENUM_CLASS_DEF(WebsocketClientState, FOR_ALL_VALUES);
-
-#undef FOR_ALL_VALUES
-
-} // namespace bts::websocket
-
-FMT_IMPL(bts::websocket::WebsocketClientState, s, "{}", s.name());

src/lib_http/http/websocket/websocket_close_status.h

@@ -1,24 +0,0 @@
-#pragma once
-
-#include "system/FMT_IMPL.h"
-#include "system/value_wrapper.h"
-
-#include <websocketpp/close.hpp>
-
-namespace bts::websocket {
-
-/**
- * Simple wrapper to provide a means implement a custom formatter for
- * `websocketpp::close::status::value` (an alias for a primitive numeric type)
- */
-using WebsocketCloseStatus = utils::ValueWrapper<
-    struct WebsocketCloseStatusTag,
-    websocketpp::close::status::value>;
-
-} // namespace bts::websocket
-
-FMT_IMPL(
-    bts::websocket::WebsocketCloseStatus,
-    s,
-    "{}",
-    websocketpp::close::status::get_string(*s));

src/lib_http/http/websocket/websocket_error_code.h

@@ -1,7 +0,0 @@
-#pragma once
-
-#include "system/FMT_IMPL.h"
-
-#include <websocketpp/common/system_error.hpp>
-
-FMT_IMPL(websocketpp::lib::error_code, ec, "{}", ec.message());

src/lib_http/http/websocket/websocketpp_exception_formatter.h

@@ -1,7 +0,0 @@
-#pragma once
-
-#include "system/FMT_IMPL.h"
-
-#include <websocketpp/error.hpp>
-
-FMT_IMPL(websocketpp::exception, e, "{}", std::string{e.what()});

src/lib_spark/CMakeLists.txt

@@ -114,8 +114,6 @@ if (DEFINED DOTNET_BINARY_DIR)
     "Copy ${PROJECT_NAME} bins to Dotnet build dir")
 endif()
 
-install(TARGETS ${PROJECT_NAME} DESTINATION "${CMAKE_INSTALL_BINDIR}")
-
 if(WIN32)
   include(${CMAKE_SOURCE_DIR}/cmake/vs_filters.cmake)
 endif()

src/lib_system/system/formatters/exception_formatter.h

@@ -1,7 +0,0 @@
-#pragma once
-
-#include "system/FMT_IMPL.h"
-
-#include <exception>
-
-FMT_IMPL(std::exception, e, "{}", std::string{e.what()});

src/lib_system/system/json/json_macros.h

@@ -1,107 +1,22 @@
-// #pragma once
-
-// #define _TO_JSON_FIELD(field) j[#field] = t.field;
-// #define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
-
-// #define SPARK_TO_JSON_IMPL(type, for_each)                                     \
-//    inline void to_json(nlohmann::json &j, const type &t) {                     \
-//       FOR_EACH(_TO_JSON_FIELD)                                                 \
-//    }
-
-// #define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
-//    inline void from_json(const nlohmann::json &j, type &t) {                   \
-//       FOR_EACH(_FROM_JSON_FIELD)                                               \
-//    }
-
-// #define OSTREAM_JSON_IMPL(type)                                                \
-//    inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
-//       nlohmann::json j;                                                        \
-//       to_json(j, t);                                                           \
-//       out << j.dump(2);                                                        \
-//       return out;                                                              \
-//    }
-
 #pragma once
 
-#include "system/json/json_reader.h"
+#define _TO_JSON_FIELD(field) j[#field] = t.field;
+#define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
 
-#define _JSON_IMPL_WRITE_JSON(name) j[#name] = src.name;
-#define TO_JSON_IMPL(type, for_all)                                            \
-   inline void to_json(nlohmann::json &j, const type &src) {                   \
-      for_all(_JSON_IMPL_WRITE_JSON)                                           \
-   }
-#define _JSON_OPTIONAL_IMPL_WRITE_JSON(field)                                  \
-   if (src.field) {                                                            \
-      j[#field] = *src.field;                                                  \
-   }
-#define TO_JSON_OPTIONAL_IMPL(type, for_all)                                   \
-   inline void to_json(nlohmann::json &j, const type &src) {                   \
-      for_all(_JSON_OPTIONAL_IMPL_WRITE_JSON)                                  \
+#define SPARK_TO_JSON_IMPL(type, for_each)                                     \
+   inline void to_json(nlohmann::json &j, const type &t) {                     \
+      FOR_EACH(_TO_JSON_FIELD)                                                 \
    }
 
-#define _JSON_IMPL_READ_JSON(name) jr.read(#name, dest.name);
-#define FROM_JSON_IMPL(type, for_all)                                          \
-   inline void from_json(const nlohmann::json &j, type &dest) {                \
-      bts::io::JsonReader jr{&j};                                    \
-      for_all(_JSON_IMPL_READ_JSON)                                            \
+#define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
+   inline void from_json(const nlohmann::json &j, type &t) {                   \
+      FOR_EACH(_FROM_JSON_FIELD)                                               \
    }
 
-#define _JSON_OPTIONAL_IMPL_READ_JSON(field)                                   \
-   jr.read_optional(#field, dest.field);
-#define FROM_JSON_OPTIONAL_IMPL(type, for_all)                                 \
-   inline void from_json(const nlohmann::json &j, type &dest) {                \
-      bts::io::JsonReader jr{&j};                                    \
-      for_all(_JSON_OPTIONAL_IMPL_READ_JSON)                                   \
-   }
-
-#define JSON_IMPL(type, for_all)                                               \
-   FROM_JSON_IMPL(type, for_all)                                               \
-   TO_JSON_IMPL(type, for_all)
-
-#define JSON_OPTIONAL_IMPL(type, for_all)                                      \
-   FROM_JSON_OPTIONAL_IMPL(type, for_all)                                      \
-   TO_JSON_OPTIONAL_IMPL(type, for_all)
-
-#ifdef LOG_JSON_READ_ERRORS
-#define JSON_READ_AS(json, field, type, dest)                                  \
-   type dest;                                                                  \
-   bts::system::json::JsonReader{&json}.read(#field, dest)
-#else
-#define JSON_READ_AS(j, field, type, dest)                                     \
-   bts::io::JsonReader{&j}.read(#field, dest);
-
-#define JSON_READ_OPTIONAL_AS(j, temp_var, field, type, dest)                  \
-   {                                                                           \
-      type temp_var{};                                                         \
-      if (bts::io::JsonReader{&j}.read(#field, temp_var)) {          \
-         dest = std::move(temp_var);                                           \
-      }                                                                        \
-   }
-#endif
-
-#define JSON_READ(json, dest, field)                                           \
-   JSON_READ_AS(json, field, decltype(dest.field), dest.field)
-#define JSON_READ_FIELDS(json, dest, fields) fields(json, dest, JSON_READ)
-
-#define JSON_READ_OPTIONAL(js, dest, field)                                    \
-   JSON_READ_OPTIONAL_AS(                                                      \
-       js, temp_var, field,                                                    \
-       std::remove_reference<decltype(dest.field.value())>::type, dest.field)
-
-#define JSON_READ_OPTIONAL_FIELDS(json, dest, fields)                          \
-   fields(json, dest, JSON_READ_OPTIONAL)
-
-#define JSON_WRITE(j, src, field) j[#field] = src.field;
-#define JSON_WRITE_FIELDS(j, src, fields) fields(j, src, JSON_WRITE)
-
-#define JSON_WRITE_OPTIONAL(j, src, field)                                     \
-   if (src.field) {                                                            \
-      j[#field] = *src.field;                                                  \
-   }
-#define JSON_WRITE_OPTIONAL_FIELDS(j, src, fields)                             \
-   fields(j, src, JSON_WRITE_OPTIONAL)
-
-#define JSON_WRITE_OPTIONAL_AS(j, field, dest)                                 \
-   if (dest) {                                                                 \
-      j[#field] = *dest;                                                       \
+#define OSTREAM_JSON_IMPL(type)                                                \
+   inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
+      nlohmann::json j;                                                        \
+      to_json(j, t);                                                           \
+      out << j.dump(2);                                                        \
+      return out;                                                              \
    }

src/lib_system/system/lock_guard.h

@@ -2,9 +2,3 @@
 
 #define LOCK_GUARD(lockable)                                                   \
    std::lock_guard<decltype(lockable)> __lock_guard_##lockable { lockable }
-
-#define BTS_LOCK_GUARD(lockable)                                               \
-   std::lock_guard<decltype(lockable)> __lock_guard { lockable }
-
-#define BTS_LOCK_GUARD_AS(lockable, name)                                      \
-   std::lock_guard<decltype(lockable)> __lock_guard_##name { lockable }

src/lib_system/system/time_span.h

@@ -53,8 +53,6 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
-constexpr TimeSpan operator-(TimeSpan lhs, TimeSpan rhs) noexcept;
-constexpr TimeSpan operator+(TimeSpan lhs, TimeSpan rhs) noexcept;
 
 std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
 
@@ -132,14 +130,6 @@ constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
    return lhs.nanos() >= rhs.nanos();
 }
 
-constexpr TimeSpan operator-(TimeSpan lhs, TimeSpan rhs) noexcept {
-   return TimeSpan{lhs.nanos() - rhs.nanos()};
-}
-
-constexpr TimeSpan operator+(TimeSpan lhs, TimeSpan rhs) noexcept {
-   return TimeSpan{lhs.nanos() + rhs.nanos()};
-}
-
 } // namespace bts::core
 
 FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());

tools/py/core/file_utils.py

@@ -27,6 +27,16 @@ def _default_build():
     return _workspace_root().joinpath('build')
 
 
+def _dotnet_runtime():
+    match sys.platform:
+        case 'win32':
+            return 'win-x64'
+        case 'linux':
+            return 'linux-x64'
+        case _:
+            raise ValueError(f'Unsupported platform {sys.platform}')
+
+
 class SparkPaths:
     @staticmethod
     def root():

tools/py/publish_api_server.py

@@ -174,15 +174,9 @@ def publish(args: PublishArgs):
     run(
         f"dotnet publish {sln} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc"
     )
-    # Copy cpp bins
-    print(f"[pseudo shell] cp -r install/{cmake_preset}/bin/* {publish_dir}")
-    if not args.dry_run:
-        copytree(f"install/{cmake_preset}/bin", publish_dir, dirs_exist_ok=True)
+    copytree(f"install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True)
     if args.output:
-        # Move to '--output' dir
-        print(f"[pseudo shell] mv {publish_dir} {args.output}")
-        if not args.dry_run:
-            os.rename(publish_dir, args.output)
+        os.rename(publish_dir, args.output)
 
 
 def _main():
@@ -192,7 +186,6 @@ def _main():
         PublishArgs(
             configuration=args.configuration,
             clean=args.clean,
-            cmake_jobs=args.cmake_jobs,
             verbosity=args.verbosity,
             dry_run=args.dry_run,
             output=args.output,

vcpkg.json

@@ -5,8 +5,7 @@
     "concurrencpp",
     "curlpp",
     "cryptopp",
-    "openssl",
-    "websocketpp"
+    "openssl"
   ],
   "builtin-baseline": "bb002e69deb36c00e9ce21810f69eff349a833f7",
   "overrides": [