Fix: Taifex Time Conversion

FIX: Taifex Crash Fixed, Added Diag Logs
ADD: TBBO & Symbol wise Naming in Recorder
2025-03-26 18:49:17 +00:00 · 2025-03-26 18:48:27 +00:00 · 2025-03-26 18:25:28 +00:00 · 2025-03-26 18:20:39 +00:00 · 2025-03-26 17:59:59 +00:00 · 2025-03-06 05:56:01 +00:00
173 changed files with 8089 additions and 1839 deletions
@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.12)
 project(spark)
+include(GNUInstallDirs)
 set(CMAKE_CXX_STANDARD 20)

 if(WIN32)
@@ -49,8 +50,6 @@ else()
  # make sparkd look for libspark.so in its directory
  set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
  set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
-  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")

  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
@@ -147,11 +146,14 @@ else()

    install(
      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
    install(
      FILES
      "${SPARK_LIB}/libACE-5.7.5.so"
      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+      "${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
+      "${SPARK_LIB}/libonixs-fix-engine.so.4"
+      "${SPARK_LIB}/libonixs-fix-engine.so"
+      DESTINATION "${CMAKE_INSTALL_BINDIR}")
  endif()
 endif()
@@ -61,7 +61,7 @@
        },
        "CMAKE_CXX_STANDARD": "20",
        "CMAKE_INSTALL_RPATH": ".",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
@@ -14,6 +14,34 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -70,6 +98,102 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -80,4 +204,19 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+		{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
+	EndGlobalSection
 EndGlobal
@@ -23,6 +23,7 @@
    <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
    <PackageReference Include="Parquet.Net" Version="5.0.1" />
    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
  </ItemGroup>

 </Project>
@@ -4,6 +4,8 @@ namespace bts.SilverPine {
   using System;

   public class NullFilterStream : ViewStream {
+      public NullFilterStream(ViewStream parent_str, string col_name)
+      : this(parent_str, col_name, true) { }
      public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
         ViewStream rs = new FilterStream(parent_str,
         r => {
@@ -0,0 +1,126 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
+   }
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}
@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("AggregationStream")]
+   public class AgregationStream : ViewStream {
+
+      public AgregationStream(ViewStream parent_str, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
+         mySetRealSource(rs);
+      }
+
+      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -2,10 +2,13 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;

 using bts;
+using bts.SilverPine;

 namespace BCG {
+   [DataStreamUserRespresentation("AggregateStream")]
   public class NonIncGroupByStream : ViewStream {
      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
         Func<string[]> _groupby_cols_func;
@@ -43,13 +46,34 @@ namespace BCG {
         _main_stream = main_stream;
         _groupby_func = groupby_func;
         _groupby_cols = _groupby_func.get_group_by_cols();
-         _row_hasher = new BlueRowHasher(_groupby_cols.Length);
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
      }

      public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }

+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+

      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }

@@ -79,7 +103,9 @@ namespace BCG {
            Log.DataFlow.info(
               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
         }
-         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
      }
   }
 }
@@ -1,10 +1,12 @@
 using bts;
+using bts.collections;

 using mbts;

 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using System.Security.Cryptography;

 namespace BCG {
@@ -35,6 +37,38 @@ namespace BCG {
         _is_tag_along = _join_spec.join_col_names.Length == 0;
      }

+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
      public void init() {
         _join_spec.stream.onChange += _on_stream_cb;
      }
@@ -43,6 +77,7 @@ namespace BCG {
         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
         if (_schema == null) {
            _schema = dp.schema;
            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
@@ -7,11 +7,12 @@ using System.Threading.Tasks;
 using mbts;

 namespace BCG {
+   // this a collection of data, theres no joining logic in here
   public class NonIncJoinSpec {
      public ViewStream stream;
      // join col names are the names on which the streams are joined (for equality),
      // data col names are the names in the original stream
-      // projected col naes are their new names, ie if its x:l_x, then x is in the 
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
      // data_col_names and l_x in projected cols
      public string[] join_col_names, data_col_names, projected_col_names;

@@ -69,4 +69,6 @@ namespace bts {
         myFireEvent(dp_out);
      }
   }
+
+
 }
@@ -13,9 +13,8 @@ namespace BCG {
   /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
   /// is preserved across the rows, ie row["id"] = f(row)["id"]
   /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
      protected IStreamMapper myMapper;
      protected IChangeableStreamMapper myChangeableMapper = null;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
@@ -0,0 +1,14 @@
+namespace bts.SilverPine
+{
+   using BCG;
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream
+   {
+      private ViewStream _parent_str;
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
+      {
+
+      }
+   }
+}
@@ -0,0 +1,10 @@
+namespace bts.SilverPine {
+   using BCG;
+
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream {
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
+      }
+   }
+}
@@ -1,9 +1,8 @@
-using Google.Protobuf.WellKnownTypes;
-
 using System;
-using BCG.Utilities.General;

-namespace BCG {
+using BCG;
+
+namespace bts.SilverPine {
   // Keeps adding changing rows to the stream, hence preserving the history
   // and creating a time series, for example this can be used to make bbo updates
   // into a time series of prices/qty 
@@ -108,6 +108,7 @@ namespace bts {

   }

+   [DataStreamUserRespresentation("Project")]
   public class NewProjectStream : ViewStream, IStreamMapper {

      private ViewStream _stream;
@@ -9,161 +9,16 @@ using BCG;
 using System.Linq;
 using System.Data.Entity;
 using System.Collections.Generic;
+using System.Data.SQLite;
+

 namespace bts.SilverPine {

-   public class DataRowWrapperRow : IBlueRow {
-      private readonly DataRow _row;
-      private readonly IBlueTableSchema _schema;
-      private readonly int _id;
-      public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
-         _row = row;
-         _id = id;
-         _schema = schema;
-      }
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual object this[string col_name] {
-         get {
-            ColumnInfo ci = _schema[col_name];
-            return this[ci.colNo];
-         }
-      }
-      public virtual object this[int index] {
-         get {
-            if (_schema[index].colName == "id")
-               return _id;
-            return _row[index];
-         }
-      }
-
-      public virtual int Count { get { return _row.ItemArray.Length; } }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
-
-      public virtual bool IsFixedSize { get { return true; } }
-
-      virtual public bool IsReadOnly { get { return true; } }
-
-      /// <summary>
-      /// The key can either be the string corrsponding to the column name or
-      /// the order of the column
-      /// </summary>
-      public virtual object this[object key] {
-         get {
-            if (key is string)
-               return this[(string)key];
-            else if (key is int)
-               return this[(int)key];
-            throw new BlueDataException("Wrong type while indexing the row");
-         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
-      }
-
-      public virtual ICollection Keys { get { return _schema.columnNames; } }
-
-      public virtual ICollection Values { get { return _row.ItemArray; } }
-
-      public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-      public virtual bool Contains(object key) { return _schema[(string)key] != null; }
-
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-   }
-
-   public class DataTableWrapperTable : IBlueRowCollection {
-
-      public class DataTableWrapperTableEnumerator : IEnumerator {
-         private IEnumerator _enumerator;
-         private IBlueTableSchema _schema;
-         private int _id = 0;
-         public DataTableWrapperTableEnumerator(DataTable table) {
-            _enumerator = table.Rows.GetEnumerator();
-            _schema = BlueDataUtility.getSchema(table);
-
-         }
-         public object Current {
-            get {
-               DataRow r = _enumerator.Current as DataRow;
-               return new DataRowWrapperRow(_schema, r, _id);
-            }
-         }
-         public bool MoveNext() {
-            _id++;
-            return _enumerator.MoveNext();
-         }
-         public void Reset() {
-            _id = 0;
-            _enumerator.Reset();
-         }
-      }
-
-      private readonly DataTable _table;
-      private IBlueTableSchema _schema;
-      private IEnumerator _enumerator;
-
-      public DataTableWrapperTable(DataTable table) {
-         _table = table;
-         _schema = BlueDataUtility.getSchema(_table);
-         _enumerator = new DataTableWrapperTableEnumerator(table);
-      }
-      public virtual IBlueRow this[int id] {
-         get {
-            // its assumed the rows index == id
-            DataRow dr = _table.Rows[id];
-            return new DataRowWrapperRow(_schema, dr, id);
-         }
-         set {
-
-         }
-      }
-
-      public virtual IBlueTableSchema schema { get { return _schema; } }
-
-      public virtual bool containsID(int id) { return id < _table.Rows.Count; }
-
-      // private List<IBlueRow> _rows() {
-      //    List<IBlueRow> rv = new List<IBlueRow>();
-      //    int id = 0;
-      //    foreach (DataRow r in _table.Rows) {
-      //       rv.Add(new DataRowWrapperRow(r, id++));
-      //    }
-      //    return rv;
-      // }
-
-      public virtual IEnumerator GetEnumerator() {
-         // var rows = _rows();
-         // return rows.GetEnumerator();
-         return _enumerator;
-      }
-
-      public virtual bool IsSynchronized { get { return false; } }
-
-      public virtual object SyncRoot { get { return this; } }
-
-      public virtual void CopyTo(Array array, int index) {
-         Log.assert(false);
-         //_rows().CopyTo(array as IBlueRow[], index); 
-      }
-
-      public virtual int Count {
-         get { return _table.Rows.Count; }
-      }
-   }
-
   public static class SQLiteHelper {
-      public static DataTable ExecuteSelectQuery(string connection_string, string query) {
+      public static string location = "C:\\Users\\m_tah\\development\\data\\";
+      public static string db_name = "stream_data.db";
+
+      public static DataTable execute_select_query(string connection_string, string query) {
         // Create a new DataTable to store the results
         DataTable dataTable = new DataTable();

@@ -183,48 +38,85 @@ namespace bts.SilverPine {
               Console.WriteLine("An error occurred: " + ex.Message);
            }
         }
-
         // Return the populated DataTable
         return dataTable;
      }
-   }

-   [DataStreamUserRespresentation("SQLite")]
-   public class SQLiteStream : ViewStream {
-      private readonly string _connection_string, _query;
-      private Stopwatch _stopwatch = new Stopwatch();
-      public SQLiteStream(string connection_string, string query) {
-         _connection_string = connection_string;
-         _query = query;
+      public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();
+
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+
+            // Begin a transaction to optimize read speed
+            using (var transaction = connection.BeginTransaction())
+            using (var command = new SQLiteCommand(query, connection, transaction))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+
+                  result.Add(row);
+               }
+
+               // Commit the transaction
+               transaction.Commit();
+            }
+         }
+
+         return result;
      }

+      public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
+         var result = new List<Dictionary<string, object>>();

-      private void _read_from_sqlite_nd_fire() {
-         _stopwatch.Start();
-         DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
-         _stopwatch.Start();
-         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
-         _stopwatch.Stop();
-         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
+         using (var connection = new SQLiteConnection(connectionString)) {
+            try {
+               connection.Open();
+
+               using (var command = new SQLiteCommand(query, connection))
+               using (var reader = command.ExecuteReader()) {
+                  while (reader.Read()) {
+                     var row = new Dictionary<string, object>();
+
+                     for (int i = 0; i < reader.FieldCount; i++) {
+                        string columnName = reader.GetName(i);
+                        object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                        row[columnName] = columnValue;
+                     }
+
+                     result.Add(row);
+                  }
+               }
+            } catch (Exception ex) {
+               Console.WriteLine("An error occurred: " + ex.Message);
+            }
+         }
+         return result;
      }

-      protected override void myOnFirstSubscriber() {
-         _read_from_sqlite_nd_fire();
+      public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
+         using (var connection = new SQLiteConnection(connectionString)) {
+            connection.Open();
+            using (var command = new SQLiteCommand(query, connection))
+            using (var reader = command.ExecuteReader()) {
+               while (reader.Read()) {
+                  var row = new Dictionary<string, object>();
+                  for (int i = 0; i < reader.FieldCount; i++) {
+                     string columnName = reader.GetName(i);
+                     object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
+                     row[columnName] = columnValue;
+                  }
+                  yield return row;
+               }
+            }
+         }
      }
-
-      protected override void myOnLastSubscriber() {
-         // this value will be cleared by the base class
-         // but in this particular stream this behaviour
-         // is not usefull since any new subscribers 
-         // would force the query to be recreated
-         //myLastDP = myLastValue;
-      }
-
   }
 }

@@ -0,0 +1,73 @@
+namespace bts.SilverPine {
+   using System;
+   using System.Data;
+   using System.Data.SQLite;
+   using System.Diagnostics;
+   using System.Collections;
+
+
+   using BCG;
+
+   [DataStreamUserRespresentation("SQLite")]
+   public class SQLiteStream : ViewStream {
+      private readonly string _connection_string, _query;
+      private Stopwatch _stopwatch = new Stopwatch();
+
+      private SQLiteStream(string connection_string, string query, bool noop) {
+         _connection_string = connection_string;
+         _query = query;
+      }
+
+      public SQLiteStream(string db_name, string query)
+      :
+      this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
+
+      public SQLiteStream(string query)
+      :
+      this(SQLiteHelper.db_name, query) {
+      }
+
+
+      private void _read_from_sqlite_and_fire() {
+         _stopwatch.Start();
+         DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         _stopwatch.Restart();
+         #region perf measures
+         // SQLiteHelper.get_table(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         // SQLiteHelper.get_table_2(_connection_string, _query);
+         // _stopwatch.Stop();
+         // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+         // _stopwatch.Restart();
+         #endregion
+         IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
+         //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+         _stopwatch.Stop();
+         Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _read_from_sqlite_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         // this value will be cleared by the base class
+         // but in this particular stream this behaviour
+         // is not usefull since any new subscribers 
+         // would force the query to be recreated
+         //myLastDP = myLastValue;
+      }
+
+   }
+
+}
@@ -80,7 +80,7 @@ namespace BCG {

      public IBlueRow map(IBlueRow row) {
         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
-         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion));
         return rv;
         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
      }
@@ -10,9 +10,11 @@ namespace bts.SilverPine {
         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour");
-            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
            return rv;
         };
         ViewStream rs = new MapStream(parent_stream,
@@ -23,9 +25,11 @@ namespace bts.SilverPine {
            MutableBlueRow rv = new MutableBlueRow(s, r);
            rv.setValue($"{inp_col}_date", dt.Date);
            rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString());
-            rv.setValue($"{inp_col}_month", dt.Date.Month.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
            rv.setValue($"{inp_col}_hour", dt.Hour);
            rv.setValue($"{inp_col}_min", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
            return rv;
         });
         mySetRealSource(rs);
@@ -39,7 +39,7 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }

-      // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2]
+      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
      // we should generlize this for any math operation
      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
         // use GlobalMapStream
@@ -91,19 +91,32 @@ namespace bts.SilverPine {
         DataPacket dp = dp_obj as DataPacket;
         List<IBlueRow> rows = dp.newRows.to_list();
         var f1_rows = rows.Where(r => _f1(r));
-         Log.assert(f1_rows.Count() == 1);
+         //Log.assert(f1_rows.Count() == 1);
         var f2_rows = rows.Where(r => _f2(r));
-         Log.assert(f2_rows.Count() == 1);
-         IBlueRow f1_r = f1_rows.ElementAt(0);
-         IBlueRow f2_r = f2_rows.ElementAt(0);
-         double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-         IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-         BlueTable out_table = new BlueTable(out_schema);
-         MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-         out_row.setValue("diff", diff);
-         out_table.addRow(out_row);
-         DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+         //Log.assert(f2_rows.Count() == 1);
+         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+            IBlueRow f1_r = f1_rows.ElementAt(0);
+            IBlueRow f2_r = f2_rows.ElementAt(0);
+            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         } else {
+            IBlueRow f1_r = rows.ElementAt(0);
+            IBlueRow f2_r = rows.ElementAt(0);
+            double diff = Double.NaN;
+            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+            BlueTable out_table = new BlueTable(out_schema);
+            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+            out_row.setValue("diff", diff);
+            out_table.addRow(out_row);
+            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }

      }
   }
@@ -1,18 +1,15 @@
 using System;
 using System.Collections.Generic;
+
 using mbts;

-namespace BCG
-{
-   internal class MutableAppendTable : IBlueRowCollection
-   {
-      private class MyRow : IBlueRow
-      {
+namespace BCG {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
         private IBlueRow myRow;
         private int myID;

-         public MyRow(IBlueRow r, int id)
-         {
+         public MyRow(IBlueRow r, int id) {
            myRow = r;
            myID = id;
         }
@@ -21,10 +18,8 @@ namespace BCG

         public IBlueTableSchema schema { get { return myRow.schema; } }

-         public object this[int index]
-         {
-            get
-            {
+         public object this[int index] {
+            get {
               if (myRow.schema[index].colName == "id")
                  return myID;
               else
@@ -52,20 +47,17 @@ namespace BCG

         public System.Collections.ICollection Keys { get { return myRow.Keys; } }

-         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }

-         public System.Collections.ICollection Values
-         {
-            get
-            {
+         public System.Collections.ICollection Values {
+            get {
               object[] vals = new object[myRow.Count];
               CopyTo(vals, 0);
               return vals;
            }
         }

-         public object this[object key]
-         {
+         public object this[object key] {
            get { return key is string ? this[key as string] : this[(int)key]; }
            set { throw new Exception("The method or operation is not implemented."); }
         }
@@ -74,8 +66,7 @@ namespace BCG

         #region ICollection Members

-         public void CopyTo(Array array, int index)
-         {
+         public void CopyTo(Array array, int index) {
            myRow.CopyTo(array, index);
            array.SetValue(myID, index + myRow.schema["id"].colNo);
         }
@@ -90,8 +81,7 @@ namespace BCG

         #region IEnumerable Members

-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-         { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }

         #endregion
      }
@@ -106,8 +96,7 @@ namespace BCG

      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }

-      public IBlueRow addRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
         if (!myIDMap.ContainsKey(tableNo))
            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -116,20 +105,17 @@ namespace BCG
         return nr;
      }

-      public IBlueRow removeRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
         myRows.deleteRow(dr);
         myIDMap[tableNo].Remove((int)r["id"]);
         return dr;
      }

-      public IBlueRowCollection removeTable(int tableNo)
-      {
+      public IBlueRowCollection removeTable(int tableNo) {
         if (!myIDMap.ContainsKey(tableNo))
            return null;
-         else
-         {
+         else {
            BlueTable rv = new BlueTable(myRows.schema);
            foreach (IBlueRow r in myIDMap[tableNo].Values)
               rv.addRow(r);
@@ -139,8 +125,7 @@ namespace BCG
         }
      }

-      public void clear()
-      {
+      public void clear() {
         myRows.clear();
         myIDMap.Clear();
      }
@@ -151,7 +136,7 @@ namespace BCG

      public IBlueTableSchema schema { get { return myRows.schema; } }

-      public bool containsID(int id) {return myRows.containsID(id);}
+      public bool containsID(int id) { return myRows.containsID(id); }

      #endregion

@@ -169,7 +154,7 @@ namespace BCG

      #region IEnumerable Members

-      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }

      #endregion
   }
@@ -179,8 +164,8 @@ namespace BCG
   /// to append (or remove) more streams at
   /// runtime
   /// </summary>
-   public class MutableAppendStream : ViewStream
-   {
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
      private bool mySubscriptionActive = false;
      private MutableAppendTable myTable;
      private int myMovingIndex;
@@ -188,22 +173,17 @@ namespace BCG
      private bool _allign_col_orders = true;

      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams))
-      { }
+         : this(new List<ViewStream>(streams)) { }

      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams))
-      { }
+         : this(allign_col_order, new List<ViewStream>(streams)) { }

      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams)
-      { }
+         : this(true, streams) { }

-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
-      {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams)
-         {
+         foreach (ViewStream vs in streams) {
            myMovingIndex++;
            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
            if (_allign_col_orders)
@@ -212,8 +192,7 @@ namespace BCG
         }
      }

-      public void addStream(ViewStream vs)
-      {
+      public void addStream(ViewStream vs) {
         myMovingIndex++;
         ///the change name is done to identify the callbacks, and the project to
         ///make all the schemas col orders the same
@@ -225,19 +204,14 @@ namespace BCG
            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      public void removeStream(ViewStream vs)
-      {
+      public void removeStream(ViewStream vs) {
         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-         {
-            if (p.Value.first == vs)
-            {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
               index = p.Key;
-               if (mySubscriptionActive)
-               {
+               if (mySubscriptionActive) {
                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null)
-                  {
+                  if (myTable != null) {
                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
                     if (delRows != null)
                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -250,15 +224,13 @@ namespace BCG
         }
      }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         mySubscriptionActive = true;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         mySubscriptionActive = false;
         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -266,20 +238,17 @@ namespace BCG
            myTable.clear();
      }

-      private void myHandleStreamChanges(object newValue)
-      {         
+      private void myHandleStreamChanges(object newValue) {
         DataPacket dp = newValue as DataPacket;
         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
         BlueTable adds = new BlueTable(myTable.schema),
                     dels = new BlueTable(myTable.schema);
         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
            IBlueRow dr = myTable.removeRow(r, index);
            dels.addRow(dr);
         }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
            IBlueRow nr = myTable.addRow(r, index);
            adds.addRow(nr);
         }
@@ -18,6 +18,7 @@ using System.Text.Json.Nodes;
 using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
+using bts;

 namespace bts {

@@ -207,6 +208,10 @@ namespace BCG {
         return rv;
      }

+      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
+         return input_schema.columnNames.Contains(col);
+      }
+
      public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         foreach (Tuple<Type, string> c in cols)
@@ -1719,8 +1724,10 @@ namespace BCG {

      public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
         List<int> rv = new List<int>();
-         foreach (string c in col_names)
+         foreach (string c in col_names) {
+            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
            rv.Add(schema[c].colNo);
+         }
         return rv.ToArray();
      }

@@ -2207,23 +2214,53 @@ namespace BCG {
         try {
            if (v == null || v == DBNull.Value)
               return error_val;
-            else {
-               if (v is DateTime)
-                  return (DateTime)v;
-               else if (v is string) {
-                  DateTime rv_dt = error_val;
-                  DateTime.TryParse((string)v, out rv_dt);
+
+            if (v is DateTime) {
+               return (DateTime)v;
+            } else if (v is string) {
+               DateTime rv_dt;
+
+               // Try parsing the string as a DateTime
+               if (DateTime.TryParse((string)v,
+                                     System.Globalization.CultureInfo.InvariantCulture,
+                                     System.Globalization.DateTimeStyles.RoundtripKind,
+                                     out rv_dt)) {
                  return rv_dt;
               } else {
-                  double oa_date = (double)Convert.ChangeType(v, typeof(Double));
-                  return DateTime.FromOADate(oa_date);
+                  return error_val; // Return the error value if parsing fails
               }
+            } else {
+               // Assume the input can be converted to a double and use FromOADate
+               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+               return DateTime.FromOADate(oa_date);
            }
         } catch {
            return error_val;
         }
      }

+
+      // private static DateTime _get_datetime_value(object v, DateTime error_val) {
+      //    try {
+      //       if (v == null || v == DBNull.Value)
+      //          return error_val;
+      //       else {
+      //          if (v is DateTime)
+      //             return (DateTime)v;
+      //          else if (v is string) {
+      //             DateTime rv_dt = error_val;
+      //             DateTime.TryParse((string)v, out rv_dt);
+      //             return rv_dt;
+      //          } else {
+      //             double oa_date = (double)Convert.ChangeType(v, typeof(Double));
+      //             return DateTime.FromOADate(oa_date);
+      //          }
+      //       }
+      //    } catch {
+      //       return error_val;
+      //    }
+      // }
+
      #endregion

      #region CSV methods
@@ -47,6 +47,11 @@ namespace BCG {
      /// </summary>
      /// <param name="assemblyName"></param>
      public void loadAssembly(string assemblyName) {
+         Func<Type, bool> is_silverpine_only = (t) => {
+            if (!assemblyName.Contains("SharedStreams.dll"))
+               return true;
+            return t.Namespace == "bts.SilverPine";
+         };
         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
         if (myAssemblyNames.Contains(assemblyName))
            return;
@@ -56,7 +61,7 @@ namespace BCG {
            Assembly.LoadFrom(assemblyName);

         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
+            if (!type.IsInterface && is_silverpine_only(type)
               && !type.IsAbstract
               && type.IsSubclassOf(typeof(ViewStream))
               && type.IsPublic) {
@@ -1,60 +1,62 @@
-"user": "bts",
-"pubsub": {
-  "milliseconds_between_sub_info_updates": 1000,
-  "milliseconds_between_time_slice_updates": 5000,
-  "milliseconds_between_no_buffer_space_retries": 100,
-  "retries_on_no_buffer_space": 1,
-  "do_it_all_seconds_between_logs": 0,
-  "heartbeat_interval": 15000,
-  "missing_heartbeats_before_gc": 50,
-  "use_managed_sockets": true,
-  "state_update_period": 5000,
-  "statistics_update_period": 5000,
-  "disposed_connection_check_period": 1000,
-  "message_processing_warning_threshold": 1000,
-  "server_port": 12345,
-  "publisher_routes": [
-    {
-      "name": "default",
-      "server_address": "bts-cloud://192.168.1.80:12345",
-      "tag": ".*"
-    }
-  ],
-  "subscriber_routes": [
-    {
-      "name": "default",
-      "server_address": "bts-cloud://192.168.1.80:12345",
-      "tag": ".*"
-    }
-  ],
-  "max_packet_size": 500000000,
-  "blue_comm_must_die": true,
-  "crushable_tags": [],
-  "tag_type_to_assembly_map": [
-    {
-      "handler_class": "BCG.PubSubServerTagHandler",
-      "assembly": "DataStreamCore.dll"
-    }
-  ],
-  "server_queue": "PubSubQueue",
-  "port_range": {
-    "first": 30400,
-    "last": 31700
-  },
-  "bind_to_random_port": false,
-  "debug_tags": "^NOTAGS$",
-  "latency_report_threshold_ms": 1500,
-  "use_substations": false,
-  "use_dynamic_clients": false,
-  "is_substation": false,
-  "internally_publish_sub_info": true,
-  "use_managed_clients_in_native_code": false,
-  "compressed_tags": "^NOTAGS$",
-  "tags_to_report_compression_ratio": "^NOTAGS$",
-  "disable_msmq": false,
-  "use_prepublished_server_state": false,
-  "ip_translation": null,
-  "gc_activation_threshold": 0,
-  "wait_period_after_gc": 1000
-}
+{
+  "user": "bts",
+  "sparkd": "http://192.168.1.80:8000/config",
+  "pubsub": {
+    "milliseconds_between_sub_info_updates": 1000,
+    "milliseconds_between_time_slice_updates": 5000,
+    "milliseconds_between_no_buffer_space_retries": 100,
+    "retries_on_no_buffer_space": 1,
+    "do_it_all_seconds_between_logs": 0,
+    "heartbeat_interval": 15000,
+    "missing_heartbeats_before_gc": 50,
+    "use_managed_sockets": true,
+    "state_update_period": 5000,
+    "statistics_update_period": 5000,
+    "disposed_connection_check_period": 1000,
+    "message_processing_warning_threshold": 1000,
+    "server_port": 12345,
+    "publisher_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://192.168.1.80:12345",
+        "tag": ".*"
+      }
+    ],
+    "subscriber_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://192.168.1.80:12345",
+        "tag": ".*"
+      }
+    ],
+    "max_packet_size": 500000000,
+    "blue_comm_must_die": true,
+    "crushable_tags": [],
+    "tag_type_to_assembly_map": [
+      {
+        "handler_class": "BCG.PubSubServerTagHandler",
+        "assembly": "DataStreamCore.dll"
+      }
+    ],
+    "server_queue": "PubSubQueue",
+    "port_range": {
+      "first": 30400,
+      "last": 31700
+    },
+    "bind_to_random_port": false,
+    "debug_tags": "^NOTAGS$",
+    "latency_report_threshold_ms": 1500,
+    "use_substations": false,
+    "use_dynamic_clients": false,
+    "is_substation": false,
+    "internally_publish_sub_info": true,
+    "use_managed_clients_in_native_code": false,
+    "compressed_tags": "^NOTAGS$",
+    "tags_to_report_compression_ratio": "^NOTAGS$",
+    "disable_msmq": false,
+    "use_prepublished_server_state": false,
+    "ip_translation": null,
+    "gc_activation_threshold": 0,
+    "wait_period_after_gc": 1000
+  }
 }
@@ -9,8 +9,90 @@ namespace bts.SilverPine {
   using ZstdSharp.Unsafe;

   public static class ContractUtils {
+      public class FutureTradableContract {
+         public string symbol;
+         public DateTime expiration;
+         public Exchange exchange;
+         public string contract_text;
+      }
+      public class FutureSpreadContract {
+         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public string symbol {
+            get {
+               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
+            }
+         }
+         public string contract_text;
+      }
+      public class FutureContinousContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      // this is for all future conytracts belonging to a symbol
+      public class FutureWildcardContract {
+         public string symbol;
+         public Exchange exchange;
+      }
+      public class FutureContract {
+         public FutureWildcardContract wildcard_contract = null;
+         public FutureTradableContract future_tradable_contract = null;
+         public FutureSpreadContract spread_tradable_contract = null;
+         public FutureContinousContract continous_constract = null;
+
+         public FutureContract(string txt) {
+            wildcard_contract = parse_wildcard_contract(txt);
+            if (wildcard_contract != null)
+               return;
+            FutureTradableContract fc = parse_future_contract(txt);
+            if (fc != null) {
+               future_tradable_contract = fc;
+               return;
+            }
+            FutureSpreadContract sc = parse_future_spread_contract(txt);
+            if (sc != null) {
+               spread_tradable_contract = sc;
+               return;
+            }
+            // this is not totally true, since ParseFutureSpreadContract does not
+            // parse ICS spreads
+            continous_constract = new FutureContinousContract() { symbol = txt };
+         }
+         public bool is_continous_contract => continous_constract != null;
+         public string symbol() {
+            if (wildcard_contract != null)
+               return wildcard_contract.symbol;
+            if (continous_constract != null) {
+               return continous_constract.symbol;
+            } else if (future_tradable_contract != null) {
+               return future_tradable_contract.symbol;
+            } else if (spread_tradable_contract != null) {
+               return spread_tradable_contract.symbol;
+            }
+            Log.assert(false);
+            return String.Empty;
+         }
+         public string contract_text {
+            get {
+               if (future_tradable_contract != null)
+                  return future_tradable_contract.contract_text;
+               if (spread_tradable_contract != null)
+                  return spread_tradable_contract.contract_text;
+               return null;
+            }
+         }
+      }
      #region Parsing
-      public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) {
+      public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
+         if (contract_txt.EndsWith("*")) {
+            contract_txt = contract_txt.Replace("*", String.Empty);
+            return new FutureWildcardContract() {
+               symbol = contract_txt,
+               exchange = exchange
+            };
+         }
+         return null;
+      }
+      public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
         // Validate the future string length
         if (!contract_txt.All(c => Char.IsLetterOrDigit(c)))
            return null;
@@ -56,23 +138,24 @@ namespace bts.SilverPine {

         // Create the FutureContract instance and set its properties
         DateTime expiration = new DateTime(year, month, 1);
-         var futureContract = new FutureContract {
+         var futureContract = new FutureTradableContract {
            symbol = symbol,
            expiration = expiration,
-            exchange = exchange
+            exchange = exchange,
+            contract_text = contract_txt
         };

         return futureContract;
      }
      // Method to parse a futures spread, e.g. "CLX4-CLZ4"
-      public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) {
+      public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
         // Split the string by '-' to handle each leg
-         var spreadParts = spreadString.Split('-');
+         var spreadParts = contract_txt.Split('-');
         if (spreadParts.Length < 2) {
            return null; // Not a valid spread format
         }

-         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureContract>>() };
+         var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };

         foreach (var part in spreadParts) {
            // Split each leg by '*', assuming the format is weight*future or just future
@@ -92,33 +175,32 @@ namespace bts.SilverPine {
            }

            // Parse the future contract
-            var futureContract = ParseFutureContract(futurePart, exchange);
+            var futureContract = parse_future_contract(futurePart, exchange);
            if (futureContract == null) {
               return null; // Failed to parse the future part
            }

            // Add the leg to the spread contract
-            spreadContract.legs.Add(new Tuple<double, FutureContract>(weight, futureContract));
+            spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
         }
-
         return spreadContract;
      }

-      public static ContractType DetermineFutureOrSpread(string contract) {
-         if (contract.Contains("-")) {
-            return ContractType.FutureSpread;
-         } else {
-            return ContractType.Future;
-         }
-      }
+      // public static ContractType DetermineFutureOrSpread(string contract) {
+      //    if (contract.Contains("-")) {
+      //       return ContractType.FutureSpread;
+      //    } else {
+      //       return ContractType.Future;
+      //    }
+      // }
      #endregion

      public static ContractType get_type(string contract_txt) {
-         FutureContract fc = ParseFutureContract(contract_txt);
+         FutureTradableContract fc = parse_future_contract(contract_txt);
         if (fc != null) {
            return ContractType.Future;
         }
-         FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt);
+         FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
         if (sc != null) {
            return ContractType.FutureSpread;
         }
@@ -133,11 +215,11 @@ namespace bts.SilverPine {
      }

      public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
-         ContractUtils.FutureContract fc = ParseFutureContract(symbol);
+         ContractUtils.FutureTradableContract fc = parse_future_contract(symbol);
         if (fc != null) {
            return Tuple.Create(ContractCategory.FutureString, fc.symbol);
         }
-         var sc = ParseFutureSpreadContract(symbol);
+         var sc = parse_future_spread_contract(symbol);
         if (sc != null) {
            return Tuple.Create(ContractCategory.FutureString, sc.symbol);
         }
@@ -165,18 +247,6 @@ namespace bts.SilverPine {
      public enum Exchange {
         CME
      }
-      public class FutureContract {
-         public string symbol;
-         public DateTime expiration;
-         public Exchange exchange;
-      }
-      public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureContract>> legs;
-         public string symbol {
-            get {
-               return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
-            }
-         }
-      }
+
   }
 }
@@ -1,35 +1,105 @@
-namespace BCG {
+namespace bts.SilverPine {
   using BCG.Utilities.General;

-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
+   using BCG;
   using Spark.Core.Reactive;

   using System;
-   using System.Collections.Concurrent;
   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
   using System.Linq;
   using System.Reactive.Linq;
   using bts.api.core;
   using Spark.Api.MarketData;

   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
+   using System.Data.Entity.Infrastructure;
+
+   public class FakeBBOStream : ViewStream, IWorker {
+      private string _symbol;
+      public BlueTable _table = new BlueTable(BBOStream._bbo_schema);
+      private DateTime _last_fired_time = DateTime.MinValue;
+      private TimeSpan _period = TimeSpan.FromMilliseconds(333);
+      private double _start_price;
+      private double _inc;
+      private int _total_count = 5;
+      private Random _rnd = new Random(DateTime.Now.Millisecond);
+      private string[] _months = new string[]{
+         "F","G", "H", "J","K", "M", "N","Q","U","V","X","Z"
+      };
+      private ViewStreamDatabase _vsd;
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc)
+      :
+      this(contract_type, exchange, symbol, start_price, inc, ViewStreamDatabase.instance) { }
+
+      public FakeBBOStream(string contract_type, string exchange, string symbol, double start_price, double inc, ViewStreamDatabase vsd) {
+         _symbol = symbol;
+         _start_price = start_price;
+         _inc = inc;
+         _vsd = vsd;
+         Log.assert(contract_type.ToLower() == "future");
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private bool _random_truefase() {
+         int r = _rnd.Next(0, 100);
+         return r % 2 == 0;
+      }
+
+      private IBlueRow _perturb_row(IBlueRow er) {
+         MutableBlueRow rv = new MutableBlueRow(er);
+         double bid_price = rv.get<double>("bid_price");
+         bid_price += _inc;
+         double ask_price = bid_price + _inc;
+         rv.setValue("bid_price", bid_price);
+         rv.setValue("ask_price", ask_price);
+         return rv;
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_fired_time < _period)
+            return;
+         _last_fired_time = DateTime.Now;
+         if (_table.Count == 0) {
+            // first fire_symbol
+            string year_code = DateTime.Now.Year.ToString().Last().ToString();
+            for (int i = 0; i < _total_count; i++) {
+               string month_code = _months[i % _months.Length];
+               string sec = $"{_symbol}{month_code}{year_code}";
+               double bid_price = _start_price + _rnd.Next(-10, 10) * _inc;
+               double ask_price = bid_price + _inc;
+               BlueRow nr = new BlueRow(_table.schema, _table.Count, _table.Count, sec,
+                                       bid_price, _rnd.Next(1, 10),
+                                       ask_price, _rnd.Next(1, 10));
+               _table.addRow(nr);
+               var dp = BlueDataUtility.build_dp(new BlueTable(_table), myLastValue as DataPacket);
+               myFireEvent(dp);
+            }
+         } else {
+            BlueTable deleted_rows = new BlueTable(_table.schema);
+            BlueTable added_rows = new BlueTable(_table.schema);
+            // go through all rows, select some at random, add them to deleted, get
+            // the replaced one and add it to added, updates table state and fire
+            for (int i = 0; i < _table.Count; i++) {
+               if (_random_truefase()) {
+                  IBlueRow er = _table[i];
+                  IBlueRow nr = _perturb_row(er);
+                  _table.deleteRow(er);
+                  _table.addRow(nr);
+               }
+            }
+            var dp = BlueDataUtility.build_dp_inc(new BlueTable(_table), myLastValue as DataPacket, true);
+            myFireEvent(dp);
+         }
+      }
+   }

   [DataStreamUserRespresentation("bbo")]
   public class BBOStream : ViewStream, IWorker {
@@ -39,7 +109,7 @@ namespace BCG {
      private int _id = 0;
      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
      MarketDataClient _md_client;
-      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+      internal static IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
                                                Tuple.Create("id", TypeCode.Int32),
                                                Tuple.Create("sec_key", TypeCode.UInt32),
                                                Tuple.Create("sec", TypeCode.String),
@@ -54,9 +124,11 @@ namespace BCG {
      }

      public BBOStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("BBOStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         // bts.Log.Gen.info("BBOStream.cons");
+         // _vsd = vsd;
+         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         mySetRealSource(rs);
      }

      protected override void myOnFirstSubscriber() {
@@ -0,0 +1,254 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Runtime.CompilerServices;
+
+   using ZstdSharp.Unsafe;
+
+   public class DateTimeRange {
+      public readonly DateTime range_beg;
+      public readonly DateTime range_end;
+      public DateTimeRange(DateTime range_beg_, DateTime range_end_) {
+         range_beg = range_beg_;
+         range_end = range_end_;
+      }
+      public static DateTimeRange parse(string range_beg_, string range_end_) {
+         DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_);
+         DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_);
+         return new DateTimeRange(range_beg, range_end);
+      }
+   }
+
+   public class FutureBarsLoader {
+      public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1day";
+                  } else
+                     return $"bars_future_{symbol}_all_1day";
+               }
+            case BarSize.ONE_MIN: {
+                  if (is_continous) {
+                     return $"bars_future_{symbol}_continous_1min";
+                  } else
+                     return $"bars_future_{symbol}_all_1min";
+               }
+         }
+         Log.assert(false);
+         return null;
+      }
+   }
+   public enum BarSize {
+      ONE_DAY = 0,
+      ONE_MIN
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
+   public class FuturesHistorical1DayBarsStream : ViewStream {
+
+      // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
+      public FuturesHistorical1DayBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         DateTime beg = dt_range.range_beg;//.ToUniversalTime(),
+         DateTime end = dt_range.range_end;//.ToUniversalTime();
+         beg = beg.Date;
+         end = end.Date.Add(new TimeSpan(23, 59, 59));
+         // this returns rows from beg and end inclusive
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
+            //TODO: filter it 
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
+   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+      // same as the above but for one min
+      // this gets the raw bars for a symbol
+      public FuturesHistorical1MinuteBarsStream(string symbol) :
+      this(symbol, DateTime.MinValue, DateTime.MaxValue) { }
+
+      // creates the non raw stream (ie for tradable contract, or continous contract)
+      public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end)
+     : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { }
+
+      // this creates a tradable contracts bars  streams or continous contracts stream. if get_all_rows is true, 
+      // it gets the raw table with all contracts for that symbol
+      public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
+                                             DateTimeRange dt_range) {
+         Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
+            string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
+            return new SQLiteStream(SQLiteHelper.db_name, sql_str);
+         };
+         string symbol = future_contract.symbol();
+         if (future_contract.wildcard_contract != null) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         }
+         if (future_contract.is_continous_contract) {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
+            ViewStream rs = get_sqlite_str(table_name);
+            mySetRealSource(rs);
+            return;
+         } else {
+            string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
+            ViewStream rs = get_sqlite_str(table_name);
+            // TODO: filter
+            rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
+            mySetRealSource(rs);
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   public class FutureHistoricalBarsStream : ViewStream {
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size_txt, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = Enum.Parse<BarSize>(bar_size_txt, true);
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+
+      public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
+         DateTime dt_range_beg = DateTime.Parse(range_beg);
+         DateTime dt_range_end = DateTime.Parse(range_end);
+         TimeSpan diff = dt_range_end - dt_range_beg;
+         BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end);
+         mySetRealSource(rs);
+      }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size)
+      : this(fut_contract, bar_size, DateTime.MinValue, DateTime.MaxValue) { }
+
+      public FutureHistoricalBarsStream(string fut_contract, string bar_size,
+                                        DateTime dt_range_beg, DateTime dt_range_end)
+                                        :
+      this(new ContractUtils.FutureContract(fut_contract), Enum.Parse<BarSize>(bar_size, true),
+         new DateTimeRange(dt_range_beg, dt_range_end)) { }
+
+      public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size,
+                                        DateTimeRange dt_range) {
+         switch (bar_size) {
+            case BarSize.ONE_DAY: {
+                  ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+            case BarSize.ONE_MIN: {
+                  ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range);
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   [DataStreamUserRespresentation("FuturePriceDiff")]
+   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
+      }
+
+      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
+      :
+      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
+      }
+
+
+      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
+         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
+
+         rs = new ColumnDifferenceStream(rs, r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt1.Equals(r_dt);
+            return rv;
+         },
+         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
+         r => {
+            DateTime r_dt = r.get_datetime("ts_event");
+            var rv = dt2.Equals(r_dt);
+            return rv;
+         }, col, col);
+         rs = new NewProjectStream(rs, "diff !");
+         mySetRealSource(rs);
+      }
+   }
+
+   // takes the contract name and expands it into multuiple columns corresponding to the contracts.
+   // the contract columns are a union, of all properties with some of them empty
+   public class ContractExpanderStream : ViewStream {
+      private static IBlueTableSchema map_schema(IBlueTableSchema schema) {
+         return schema.add_column(TypeCode.String, "contract_type");
+      }
+      public ContractExpanderStream(ViewStream parent_str, string contract_col_name) {
+         ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema),
+         r => {
+            string contract = r.get<string>(contract_col_name);
+            var ct = ContractUtils.get_type(contract);
+            MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r);
+            rv.setValue("contract_type", ct.ToString());
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+
+}
@@ -0,0 +1,53 @@
+using BCG;
+
+namespace bts.SilverPine {
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+          r=contractExpander(r,"symbol")
+          r=filter(r,"contract_type='FutureSpread'")
+          r=aggregate(r,"symbol","volume:Sum","ts_event")
+          r=aggregate(r,"volume_sum:Max")
+          r=newProject(r,"volume_sum_max:volume symbol !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          symbol                          System.String
+          id                              System.Int32
+          volume                          System.Double
+   */
+
+   [DataStreamUserRespresentation("VolumeAgg")]
+   public class VolumeAggregationStream : ViewStream {
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+      :
+      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+
+      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+         ViewStream r;
+         r = new FutureHistoricalBarsStream(future_contract,
+                                             range_beg.ToString(), range_end.ToString());
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, $"contract_type='{contract_type}'");
+         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+         r = new NonIncGroupByStream(r, "volume_sum:Max");
+         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+         mySetRealSource(r);
+      }
+   }
+
+   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+   public class PeriodicVolumeAggregationStream : ViewStream {
+      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+                                             TimeSpan period, ViewStreamDatabase db) {
+      }
+   }
+}
@@ -1,68 +1,117 @@
+// namespace bts.SilverPine {
+//    using bts;
+//    using BCG;
+//    using MongoDB.Driver.Linq;
+//    using System.ComponentModel.DataAnnotations;
+//    using ZstdSharp.Unsafe;
+
+//    [DataStreamUserRespresentation("VWAP")]
+//    public class VWAPStream : ViewStream {
+//       // returns the VWAP stream for each contract 
+//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+//          // stream is grouped per contract and then vwap is calculated per contract
+//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+//          (rows, idx) => {
+//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+//             IBlueRow ir = rows.ElementAt((int)idx);
+//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+//             //vwap = -313.987;
+//             nr.setValue("vwap", vwap);
+//             return nr;
+//          });
+//          return rs;
+//       }
+//       // the args want dt1 > dt2, if its nto we, swap them
+//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//          if (dt1 < dt2) {
+//             DateTime aux = dt1;
+//             dt1 = dt2;
+//             dt2 = aux;
+//          }
+//       }
+
+//       // this assumes the ohlc_str has diff contracts
+//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+//          swap_if_necessary(ref begin_dt, ref end_dt);
+//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+//          string symbol = String.Empty;
+//          symbol = ccat_sym_contr.Item2;
+//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+//          ViewStream rs = vwap_str;
+//          switch (ccat_sym_contr.Item1) {
+//             case ContractUtils.ContractCategory.FutureString:
+//             case ContractUtils.ContractCategory.SpreadString:
+//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+//                break;
+//             case ContractUtils.ContractCategory.Symbol:
+//                rs = new FuturesActiveContractStream(vwap_str);
+//                break;
+//          }
+//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+//          rs = new NewProjectStream(rs, "! vwap");
+//          mySetRealSource(rs);
+//       }
+
+//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+//       :
+//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+//    }
+// }
+
+using System;
+
+using BCG;
+
+using bts.api;
+
 namespace bts.SilverPine {
-   using bts;
-   using BCG;
-   using MongoDB.Driver.Linq;
-   using System.ComponentModel.DataAnnotations;
-   using ZstdSharp.Unsafe;
+   /*
+      ----------------------------------------------------------------------
+                                  *Expression*
+      ----------------------------------------------------------------------
+          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+          r=math(r, "vprice = volume * close")
+          r=aggregate(r, "vprice:Sum, volume:Sum")
+          r=math(r, "vwap = vprice_sum / volume_sum")
+          r=newProject(r, "vwap !")
+
+      ----------------------------------------------------------------------
+                                    *Schema*
+      ----------------------------------------------------------------------
+          id                              System.Int32
+          vwap                            System.Double
+   */

   [DataStreamUserRespresentation("VWAP")]
   public class VWAPStream : ViewStream {
-      // returns the VWAP stream for each contract 
-      private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-         ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-         // stream is grouped per contract and then vwap is calculated per contract
-         GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-         (rows, idx) => {
-            Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-            IBlueRow ir = rows.ElementAt((int)idx);
-            var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-            var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-            double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-            IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-            MutableBlueRow nr = new MutableBlueRow(schema, ir);
-            //vwap = -313.987;
-            nr.setValue("vwap", vwap);
-            return nr;
-         });
-         return rs;
-      }
-      // the args want dt1 > dt2, if its nto we, swap them
-      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-         if (dt1 < dt2) {
-            DateTime aux = dt1;
-            dt1 = dt2;
-            dt2 = aux;
-         }
-      }
-
-      // this assumes the ohlc_str has diff contracts
-      public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         swap_if_necessary(ref begin_dt, ref end_dt);
-         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-         string symbol = String.Empty;
-         symbol = ccat_sym_contr.Item2;
-         var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-         ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-         // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-         // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-         ViewStream rs = vwap_str;
-         switch (ccat_sym_contr.Item1) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(vwap_str);
-               break;
-         }
-         rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-         rs = new NewProjectStream(rs, "! vwap");
-         mySetRealSource(rs);
-      }
-
-      public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
      :
-      this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+         // since this is already filtered, we can groupBy iit without a groupBy column
+         r = new MathStream(r,
+                 "vprice = volume * close");
+         r = new NonIncGroupByStream(r,
+                                      "vprice:Sum, volume:Sum");
+         r = new MathStream(r,
+                             "vwap = vprice_sum / volume_sum");
+         r = new NewProjectStream(r,
+                                   "vwap !");
+         mySetRealSource(r);
+      }
   }
-}
+}
@@ -75,4 +75,31 @@ namespace bts.SilverPine {
         mySetRealSource(rs);
      }
   }
+}
+
+
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
+         mySetRealSource(r);
+      }
+   }
 }
@@ -2,15 +2,14 @@ namespace bts.SilverPine {
   using bts;
   using BCG;

-   // it uses the volume - better to use open interst perhap - to detect the rollover, is when a new 
-   // futuere exp becomes the highest volume one
+   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+   // future exp becomes the highest volume one
   [DataStreamUserRespresentation("FuturesRoll")]
   public class FuturesRollStream : ViewStream {
      public FuturesRollStream(string future_symbol) {
-         //ViewStream ohlcv_str = new OHLCVStream(future_symbol);
-         ViewStream rs = new FuturesActiveContractStream(future_symbol);
-         rs = new ColumnEvolutionStream(rs, "ts_event", "contract");
-         rs = new NewProjectStream(rs, "ts_event:roll_date !");
+         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
         mySetRealSource(rs);
      }
   }
@@ -2,6 +2,93 @@ namespace bts.SilverPine {
   using bts;
   using BCG;
   using MongoDB.Driver;
+
+   // OHLCV request breakdown
+   // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it)
+   // bar_size (one min, one day)
+   // live/historic
+   // time range
+
+   // internal code oranization only
+   public class OHLCV_1min_stream : ViewStream {
+
+      public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) :
+      this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { }
+
+      public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) {
+         Func<string, bool, SQLiteStream> get_sqlite_str = (symbol, is_continous_contracts) => {
+            string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min";
+            string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3";
+            string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
+               range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss");
+            string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
+            return new SQLiteStream(db_str, sql_str);
+         };
+
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
+         var contract_cat = tup.Item1;
+         switch (contract_cat) {
+            case ContractUtils.ContractCategory.Symbol: {
+                  ViewStream res = get_sqlite_str(tup.Item2, true);
+                  mySetRealSource(res);
+                  // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it
+                  // ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false);
+                  // // the following is whats written to the DB in the _contionous table
+                  // ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event");
+                  // ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2);
+                  // ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume",
+                  //                                     active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract");
+                  // res = new NullFilterStream(res, "continous_contract", true);
+                  // res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !");
+                  // mySetRealSource(res);
+                  break;
+               }
+            default: {
+                  ViewStream sql_str = get_sqlite_str(tup.Item2, false);
+                  ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'");
+                  // filter here
+                  mySetRealSource(rs);
+                  break;
+               }
+         }
+      }
+   }
+
+   // this is to make bigger bars form smaller bars. Right now the assumption it only works
+   // with symbols vs contracts, and the 1min <= aggregation_period < 1day
+   [DataStreamUserRespresentation("OHLCVAggregated")]
+   public class OHLCVAggregatedStream : ViewStream {
+      public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) :
+      this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) {
+      }
+
+      public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) {
+         Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol);
+         Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol);
+         if (aggregation_period < TimeSpan.FromDays(1.0) &&
+            aggregation_period > TimeSpan.FromMinutes(1.0)) {
+            ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end);
+            r = new DateTimeExpanstionStream(r,
+                                              "ts_event");
+            double tot_secs = aggregation_period.TotalSeconds;
+            r = new MathStream(r,
+                                $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})");
+            r = new NonIncGroupByStream(r,
+                                         "bucket",
+                                         // column:operation
+                                         "open:first, high:max, low:min, close:last, volume:sum",
+                                         "ts_event");
+            r = new SortStream(r,
+                                "-ts_event");
+            mySetRealSource(r);
+         } else {
+            // fix it later
+            Log.assert(false);
+         }
+      }
+   }
+
+
   [DataStreamUserRespresentation("OHLCV")]
   // This stream retreives the OHLCV table as is, and fires it as-is. 
   // The only logic here is as follows:
@@ -80,12 +167,13 @@ namespace bts.SilverPine {
         switch (ohlcv_params.type) {
            case OHLCVType.LIVE:
            case OHLCVType.LIVE_AND_HISTORIC:
+               // TODO
               Log.assert(false);
               break;
            case OHLCVType.HISTORIC:
               // TODO: assuming 1 day only for historic               
               string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
-               //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
+                                                                                         //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
               string db_str = $"Data Source={_location}{_db_name};Version=3";
               string sql_str = $"select * from {table_name}";
               // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
@@ -111,6 +199,11 @@ namespace bts.SilverPine {
      :
      this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }

+      // aggregates the bars
+      public OHLCVStream(string symbol, TimeSpan agregation_period) {
+
+      }
+


   }
@@ -2,50 +2,23 @@ namespace bts.SilverPine {
   using BCG;

   using bts;
-
-   using DnsClient.Protocol;
-
-   using MongoDB.Driver;
-
+   // used directly in blueorb
   [DataStreamUserRespresentation("Settlement")]
   public class SettlementPriceStream : ViewStream {

-      public SettlementPriceStream(string future_name, string date_txt)
-      : this(future_name, date_txt, String.Empty) { }
+      // public SettlementPriceStream(string future_name, string date_txt)
+      // : this(future_name, date_txt, date_txt) { }

-      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }

-      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
-      : this(symbol_or_contract, begin_dt, DateTime.MinValue) { }
+      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+      : this(future_contract, begin_dt, begin_dt) { }

-      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         // DateTime begin_dt = DateTime.Parse(begin_dt_txt);
-         // DateTime end_dt = String.IsNullOrEmpty(end_dt_txt) ? DateTime.MinValue : DateTime.Parse(end_dt_txt);
-         Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-         ContractUtils.ContractCategory contr_cat = ccat_sym_contr.Item1;
-         string symbol = ccat_sym_contr.Item2;
-         ViewStream settl_str = new OHLCVStream(symbol);
-         // TODO: close != settlement, get settlement from somewhere
-         ViewStream rs = new NewProjectStream(settl_str, "close:settlement");
-         switch (contr_cat) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(settl_str, r => r.get<string>("contract").CompareTo(symbol_or_contract) == 0);
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(settl_str);
-               break;
-         }
-         if (DateTime.MinValue.Equals(end_dt)) {
-            rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-         }
-         rs = new FilterStream(rs, r => {
-            DateTime dt = r.get_datetime("ts_event");
-            return dt <= begin_dt && dt >= end_dt;
-         });
-         rs = new MathStream(rs, "settlement = close * 1.0");
-         rs = new NewProjectStream(rs, " ! ts_event:date settlement");
+      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
+         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
         mySetRealSource(rs);
      }

@@ -0,0 +1,276 @@
+# **DSL Master Learning Prompt**
+
+This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
+
+# **Introduction to Streams**
+
+The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
+
+# **Streams**
+
+## **FutureHistoricalBars (DataSource Stream)**
+
+The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
+
+#### **Stream arguments (parameters):**
+
+1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
+1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
+2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
+   For SIlver March April 2019 Silver Future Calendar spread)  
+3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
+2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
+   * `ONE_DAY` – Daily bars.  
+     * `ONE_MIN` – Minute bars.  
+3. **Optional:**  
+   * **`range_begin` (datetime):** Start of the desired time range.  
+   * **`range_end` (datetime):** End of the desired time range.
+
+#### **Schema:**
+
+The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
+
+| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
+| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
+| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
+|      |                       |          |         |         |         |         |        |
+
+# **FilterStream (Operator Stream)**
+
+Filters rows of a parent stream based on conditions applied to its columns.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
+2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
+
+#### **Examples:**
+
+1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
+2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
+3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
+   
+
+**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
+
+# **ProjectStream (Operator Stream)**
+
+Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
+
+#### **Stream arguments (parameters):**
+
+3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
+4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
+
+#### **Examples:**
+
+	The following DSL renames the open column to O, high to H, low to L and close to C  
+**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
+
+# **GroupByStream (Operator Stream)**
+
+The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
+2. **Group By Column (string, optional):**
+
+   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
+
+3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   Find the total SP 500 volume executed in the first week of 2021\.  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = GroupByStream(bars,”volume_Sum”)`  
+2. **NL:**   
+   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = GroupByStream(bars,”high:Max low:Min”)`  
+   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
+3. **NL:**   
+   **Find the Average daily volume per contract in Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
+   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
+
+# **SingleRowMathStream (Operator Stream)**
+
+The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):**The source stream provides the base data  
+2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
+
+#### **Examples:**
+
+4. **NL:**   
+   Find the daily price movement in the SP 500 Futures through the month of Jan  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
+   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
+5. **NL:**   
+   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
+
+#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
+
+**Schema Impact:**
+
+# **MultiRowRollingMathStream (Operator Stream)**
+
+The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
+2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
+3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
+4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   **Find the 20 d MA of SP 500 Futures close prices**  
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
+2. **NL:**   
+   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
+
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
+   `// calculating typical_price`
+
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
+   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
+
+   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
+
+3. **NL:**   
+   **Find the Average daily volume of Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
+   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
+
+# **UnionStream (Operator Stream)**
+
+Like the SQL Union appends all the rows from 2 streams together.
+
+#### **Stream arguments (parameters):**
+
+5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
+6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
+
+#### **Examples:**
+
+4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
+   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
+   
+
+#### **Schema Impact:**
+
+None
+
+# **DateTimeExpanderStream (Operator Stream)**
+
+DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
+2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
+
+**New Columns:**  
+`<input_dt_col>_date` (Date of the value, without the time part)
+
+* `<input_dt_col>_day,` (Day of the week)  
+* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
+* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
+* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
+* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
+* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
+
+#### **Examples:**
+
+**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
+**Input Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
+
+**Output Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
+
@@ -2,9 +2,13 @@
   "prompts": [
      {
         "category": "Settlement",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however you should use the Today's real date",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or ESZ9 etc. The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for a range pf dates from begin_dt to end_dt. You will need to convert statements  expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August, First day of last quarter etc to actual dates. The function Settlement's second and third arguments are dates, so never pass a string like FirstDayOfLastQUarter. You will need to know what Today is, since there will be references to last week, yesterday etc.",
         "prompt": "What was the settlement for SP 500 Future on June 8th?",
         "examples": [
+            {
+               "natural_language": "What was the settlement for SP 500 Future on June 8th?",
+               "dsl": "Settlement(\"ES\", 6/8/2023)"
+            },
            {
               "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
               "dsl": "Settlement(\"ES\", 4/22/2023)"
@@ -41,70 +45,82 @@
      },
      {
         "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, period_start_date, period_end_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the start and end of the VWAP period, chronologically. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/05/2023  to 10/15/2023 (to cover for 10 days going back from the given date). so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
         "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
         "examples": [
            {
               "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.",
-               "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)"
+               "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
            },
            {
-               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-               "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)"
+               "natural_language": "Find the 10 day VWAP of the Crude Oil Future on 10/10/2023.",
+               "dsl": "VWAP(\"CL\", 10/01/2023, 10/10/2023)"
            },
            {
-               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-               "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)"
+               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future on 9/11/2023.",
+               "dsl": "VWAP(\"ZN\", 8/11/2023, 9/11/2023)"
            },
            {
               "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
-               "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)"
+               "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
+            },
+            {
+               "natural_language": "Find the 3 month VWAP of the SP 500 futures on 10/10/2023",
+               "dsl": "VWAP(\"ES\", 7/10/2023, 10/10/2023)"
            }
         ]
      },
      {
-         "category": "Price Diff",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024",
-         "prompt": "What is the difference in  SP 500 Future from July 10th to July 14th?",
+         "category": "FuturePriceDiff",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date (I am not providing any function to find this date, you have to find the nearest trading date yourself), for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th? (or how much did the SP 500 Future price move from July 10th to July 14th?)",
         "examples": [
            {
               "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
-               "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)"
+               "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
            },
            {
-               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-               "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)"
+               "natural_language": "What is the difference in SP 500 Future price from the first day of the first quarter of 2020 to the last day of the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+            },
+            {
+               "natural_language": "How much did SP 500 Futures move in the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
            },
            {
               "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
-               "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)"
+               "dsl": "FuturePriceDiff(\"NGZ4\", 8/20/2024, 8/30/2024)"
+            },
+            {
+               "natural_language": "What is the price move in  SP 500 Future from Aug 30th 2023 to October 10th 2023",
+               "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
+            },
+            {
+               "natural_language": "How much did  SP 500 Future move during the first quarter of last year",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
            }
         ]
      },
      {
-         "category": "Absolute Volume",
-         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ",
-         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+         "category": "Contract Aggregate Volume",
+         "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
+         "prompt": "Which  SP 500 Future traded the most between 7/10/2024 and 7/14/2024",
         "examples": [
            {
-               "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024",
-               "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)"
+               "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 7/10/2024, 7/14/2024)"
            },
            {
-               "natural_language": "Which Gold Spread traded the most between Today and Sept 20th",
-               "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)"
+               "natural_language": "Which Gold Spread traded the most in the first month of this year",
+               "dsl": "VolumeAgg(\"GC\", \"FutureSpread\", 1/1/2024, 1/31/2024)"
            },
            {
-               "natural_language": "Which 10 year Spread traded the highest volume last week?",
-               "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)"
+               "natural_language": "Which 10 year Spread traded the highest volume last week to now?(in this example(only) we are assuming today is 11/26/2024)",
+               "dsl": "VolumeAgg(\"ZN\", \"FutureSpread\", 11/19/2024, 11/26/2024)"
            },
            {
-               "natural_language": "Which Corn Spread traded the most today?",
-               "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)"
-            },
-            {
-               "natural_language": "Which Gold Future traded the most today?",
-               "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)"
+               "natural_language": "Which Corn Spread traded the most today (in this example(only) we are assuming today is 11/26/2024)?",
+               "dsl": "VolumeAgg(\"ZC\", \"FutureSpread\", 11/26/2024, 11/26/2024)"
            }
         ]
      },
@@ -114,8 +130,8 @@
         "prompt": "When does the roll typically occur In  SP 500?",
         "examples": [
            {
-               "natural_language": "When does the roll typically occur In CL?",
-               "dsl": "FuturesRoll(\"CL\")"
+               "natural_language": "When does the roll typically occur In SP 500?",
+               "dsl": "FuturesRoll(\"ES\")"
            },
            {
               "natural_language": "When does the roll typically occur for 10 year treasury?",
@@ -159,8 +175,8 @@
         ]
      },
      {
-         "category": "Average Volume",
-         "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+         "category": "Open Interest",
+         "description": "This is a few shot learning example for an NL to DSL.",
         "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
         "examples": [
            {
@@ -218,6 +234,17 @@
               "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")"
            }
         ]
+      },
+      {
+         "category": "Price Movement",
+         "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking for the average price movement for e-minis (aka ES aka SP Futures aka SP 500 Futures)) on Wednesday between 9:30 am and 9:40 AM?  the multiline DSL is as follows (along with the explaination in comments before each line, dont include the comments in the asnwer):  //following line gets the 1 min ohlcv bars of ES \n r=OHLCV_1min_stream(\"ES\")\n //this passes it to the filter stream, which filters the rows for Wednesday, and time bw 9:30 and 9:40.\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\n// this aggregates it using the date as the agregation key, it find the max of the column called high, and the low of the column called min, to find the max and the min prices in that time window\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\n/// this then passes the previous result to the math stream operator which calculates the move, which is the difference between high_max and low_min (columns that were added from the previous aggregate)\nr=math(r,\"move = high_max - low_min\")\n// At the end we compute the mean of the move column, notice this is the total mean, so we are not providing an aggregation col to agregate\nr=aggregate(r, \"move:mean\")\n// we project out every col by the ! operator and call rename move_mean as average_vol\nr=newProject(r,\"move_mean:average_move !\")",
+         "prompt": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM ?",
+         "examples": [
+            {
+               "natural_language": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM?",
+               "dsl": "r=OHLCV_1min_stream(\"ES\")\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\nr=math(r,\"move = high_max - low_min\")\nr=aggregate(r, \"move:mean\")\nr=newProject(r,\"move_mean:average_move !\")"
+            }
+         ]
      }
   ]
 }
@@ -1,4 +1,4 @@
-namespace sbts.SilverPine {
+namespace bts.SilverPine {
   using BCG;

   using bts;
@@ -105,4 +105,4 @@ add_custom_command(

 add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)

-install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}")
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -74,6 +74,11 @@ type TaifexExchangeField =
  | Origin of CustomerOrFirm
  | ClearingAccount of string

+type TwseExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -800,6 +800,7 @@ let venueToString (venue:bts.api.Venue) =
    | bts.api.Venue.FTX -> "FTX"
    | bts.api.Venue.OKX -> "OKX"
    | bts.api.Venue.TAIFEX -> "TAIFEX"
+    | bts.api.Venue.TWSE -> "TWSE"

 let exchange_display expirationNameList =
  let s = sprintf "%O" expirationNameList.product.exchange
@@ -138,7 +138,7 @@
         _dispose = Disposable.Create(resources.ToArray());

         static string fmt_sec(TimeAndSalesRowVM row) => row.security.ToString();
-         static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss tt");
+         static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss.ffffff");
         static string fmt_price(TimeAndSalesRowVM row) => row.security.spec_or_default().format(row.price);
         static string fmt_aggressor(TimeAndSalesRowVM row) => row.aggressor_side.ToString();
         static string fmt_volume(TimeAndSalesRowVM row) => new TickFactors(1, 100).format(row.volume);
@@ -37,7 +37,7 @@
    <PackageReference Include="NAudio" Version="2.0.1" />
    <PackageReference Include="NUnit" Version="3.13.3" />
    <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
    <PackageReference Include="OxyPlot.Core" Version="2.1.2" />
    <PackageReference Include="OxyPlot.Wpf" Version="2.1.2" />
    <PackageReference Include="protobuf-net" Version="3.2.26" />
@@ -1,41 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-
-// namespace BCG
-// {
-//    public class DateTimeFormatter
-//    {
-//       private delegate string MyFormatDateTime (DateTime v);
-//       public enum ExpirationShortForms
-//       {
-//          DayMonthYearDiff2000,
-//          MonthYearDiff2000
-//       }
-//       public enum DateTimeFormats { Regular, ShortDate, DayMonthYearDiff2000, MonthYearDiff2000 }
-//       private static Dictionary<string, MyFormatDateTime> _local_formats = null;
-//       private static void _init()
-//       {
-//          if (_local_formats == null)
-//          {
-//             _local_formats = new Dictionary<string, MyFormatDateTime>();
-//             _local_formats.Add(DateTimeFormats.Regular.ToString(), v => v.ToString());
-//             _local_formats.Add(DateTimeFormats.ShortDate.ToString(), v => v.ToShortDateString());
-//             _local_formats.Add(DateTimeFormats.DayMonthYearDiff2000.ToString(),
-//                (DateTime v) => { return String.Format("{0} {1}{2}", v.Day, BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); });
-//             _local_formats.Add(DateTimeFormats.MonthYearDiff2000.ToString(),
-//                (DateTime v) => { return String.Format("{0}{1}", BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); });            
-//          }
-//       }
-//       public static string format(DateTime val, string format)
-//       {
-//          _init ();
-//          MyFormatDateTime f = null;
-//          if (_local_formats.TryGetValue(format, out f))
-//             return f(val);
-//          else
-//             return val.ToString(format);
-//       }
-//    }
-// }
@@ -46,7 +46,7 @@
            Filled = 56,
            Symbol = 60,
            Time = 112,
-            TimeOfDay = 64,
+            TimeOfDay = 84,
            TimeInForce = 47,
            User = 53,
            Aggressor = 80,
@@ -69,8 +69,16 @@ namespace mbts {
      public static bool is_valid(this RegionalTickFactors regional_tick) =>
         regional_tick.low_tick.is_valid() && regional_tick.high_tick.is_valid();

-      public static bool is_integer(this TickFactors tick) =>
-         tick.tick_unit == tick.tick_base;
+      public static bool is_integer(this TickFactors tick) {
+         if (tick.tick_unit == tick.tick_base) {
+            return true;
+         }
+         if (tick.tick_unit > tick.tick_base && tick.tick_base != 0) {
+            var units_per_base = (int)(tick.tick_unit / tick.tick_base);
+            return units_per_base * tick.tick_base == tick.tick_unit;
+         }
+         return false;
+      }

      public static TickFactors tick_factors(this RegionalTickFactors regional_tick, double value) =>
         value > regional_tick.low_tick_inclusive_end ?
@@ -295,6 +295,9 @@ namespace mbts.spark {
         var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit);
         // if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar
         if (ticks_per_row > ticks_per_dollar) {
+            if (ticks_per_dollar == 0) {
+               return null;
+            }
            return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar)));
         } else if (ticks_per_row == ticks_per_dollar) {
            return ticks_per_row;
@@ -111,6 +111,7 @@ namespace bts.api {
            Security.ExchangeDataOneofCase.Okx => security.Okx.SecurityDesc,
            Security.ExchangeDataOneofCase.Cboe => security.Cboe.SecurityDesc,
            Security.ExchangeDataOneofCase.Taifex => security.Taifex.SecurityDesc,
+            Security.ExchangeDataOneofCase.Twse => security.Twse.SecurityDesc,
            _ => throw new NotImplementedException("security_desc not implemented for exchange")
         };

@@ -158,6 +159,7 @@ namespace bts.api {
            domain.Venue.Okx => Venue.Okx,
                        domain.Venue.Cboe => Venue.Cboe,
            domain.Venue.Taifex => Venue.Taifex,
+            domain.Venue.Twse => Venue.Twse,


            domain.Venue.Unspecified => Venue.Unset,
@@ -178,6 +180,7 @@ namespace bts.api {
            Venue.Okx => domain.Venue.Okx,
            Venue.Cboe => domain.Venue.Cboe,
            Venue.Taifex => domain.Venue.Taifex,
+            Venue.Twse => domain.Venue.Twse,

            Venue.Unset => domain.Venue.Unspecified,
            _ => throw new NotImplementedException(),
@@ -0,0 +1,61 @@
+{
+  "user": "bts",
+  "pubsub": {
+    "milliseconds_between_sub_info_updates": 1000,
+    "milliseconds_between_time_slice_updates": 5000,
+    "milliseconds_between_no_buffer_space_retries": 100,
+    "retries_on_no_buffer_space": 1,
+    "do_it_all_seconds_between_logs": 0,
+    "heartbeat_interval": 15000,
+    "missing_heartbeats_before_gc": 50,
+    "use_managed_sockets": true,
+    "state_update_period": 5000,
+    "statistics_update_period": 5000,
+    "disposed_connection_check_period": 1000,
+    "message_processing_warning_threshold": 1000,
+    "server_port": 12345,
+    "publisher_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://localhost:12345",
+        "tag": ".*"
+      }
+    ],
+    "subscriber_routes": [
+      {
+        "name": "default",
+        "server_address": "bts-cloud://localhost:12345",
+        "tag": ".*"
+      }
+    ],
+    "max_packet_size": 500000000,
+    "blue_comm_must_die": true,
+    "crushable_tags": [],
+    "tag_type_to_assembly_map": [
+      {
+        "handler_class": "BCG.PubSubServerTagHandler",
+        "assembly": "DataStreamCore.dll"
+      }
+    ],
+    "server_queue": "PubSubQueue",
+    "port_range": {
+      "first": 30400,
+      "last": 31700
+    },
+    "bind_to_random_port": false,
+    "debug_tags": "^NOTAGS$",
+    "latency_report_threshold_ms": 1500,
+    "use_substations": false,
+    "use_dynamic_clients": false,
+    "is_substation": false,
+    "internally_publish_sub_info": true,
+    "use_managed_clients_in_native_code": false,
+    "compressed_tags": "^NOTAGS$",
+    "tags_to_report_compression_ratio": "^NOTAGS$",
+    "disable_msmq": false,
+    "use_prepublished_server_state": false,
+    "ip_translation": null,
+    "gc_activation_threshold": 0,
+    "wait_period_after_gc": 1000
+  }
+}
@@ -1,4 +1,7 @@
 namespace spark_api_server {
+   using BCG;
+
+   using Bts.PubSub;
   using Bts.Websocket;

   using bts;
@@ -29,8 +32,11 @@

      static async Task<int> _run(RunOptions opts) {
         Console.WriteLine("Starting spark_api_server... ");
+         opts.MergeWithConfigFile();
+         opts.MergeWith(RunOptions.Default());
         Log.notnull(opts.user);
         Log.notnull(opts.password);
+         PubSubConfig.SetGlobalConfig(Log.notnull(opts.PubSub));
         var service_ctx = ServiceContext.create(opts.sparkd);
         await service_ctx.login(opts.user, opts.password);

@@ -67,6 +73,14 @@
            }

         });
+         var vsdTask = Task.Run(() => {
+            Log.Gen.info("Starting ViewStreamDatabase work loop");
+            var cancellationToken = cancellationTokenSource.Token;
+            while (!cancellationToken.IsCancellationRequested) {
+               ViewStreamDatabase.instance.doWork();
+               Thread.Sleep(100);
+            }
+         }, cancellationTokenSource.Token);
         // var task = Task.Run(() => {
         //    while (!cancellationTokenSource.IsCancellationRequested) {
         //       // Do something that can be interrupted
@@ -76,8 +90,11 @@
         // Console.WriteLine("Press any key to stop the server...");
         // Console.ReadKey();
         // cancellationTokenSource.Cancel();
-         await Task.WhenAny(listen);
+         await Task.WhenAny(listen, vsdTask);
         //UPTILL HERE
+         // These tasks aren't designed to finish naturally. If the code reaches here, it means one
+         // of them threw an exception. Cancel the other task to exit (somewhat) gracefully.
+         cancellationTokenSource.Cancel();
         return (int)errors;
      }
   }
@@ -1,6 +1,13 @@
 namespace Spark.Api.Server {
+   using Bts.PubSub;
+
   using CommandLine;

+   using System;
+   using System.IO;
+   using System.Text.Json;
+   using System.Text.Json.Serialization;
+
   [Verb("run")]
   public class RunOptions {
      [Option("user", Default = CommonOptionMetadata.DEFAULT_USER)]
@@ -14,5 +21,44 @@ namespace Spark.Api.Server {
         MetaValue = CommonOptionMetadata.SPARKD_META_VALUE,
         HelpText = CommonOptionMetadata.SPARKD_HELP_TEXT)]
      public string? sparkd { get; set; }
+
+      [JsonPropertyName("pubsub")]
+      public PubSubConfig? PubSub { get; set; }
+
+      [JsonIgnore]
+      [Option(
+         "config",
+         MetaValue = "PATH",
+         HelpText = "Path to json file representing these options (RunOptions)")]
+      public string? ConfigFile { get; set; }
+
+      public static RunOptions Default() =>
+         new() {
+            PubSub = PubSubConfig.Default(),
+         };
+
+      public void MergeWith(RunOptions other) {
+         sparkd ??= other.sparkd;
+         PubSub ??= other.PubSub;
+         ConfigFile ??= other.ConfigFile;
+      }
+
+      /// <summary>
+      /// Merges options with those defined in `ConfigFile` (does not overwrite
+      /// non-null existing values).
+      /// </summary>
+      /// <exception cref="InvalidOperationException">
+      /// JsonSerializer.Deserialize returns null
+      /// </exception>
+      public void MergeWithConfigFile() {
+         if (ConfigFile is null) {
+            return;
+         }
+         var cfgString = File.ReadAllText(ConfigFile);
+         var parsedCfg = JsonSerializer.Deserialize<RunOptions>(cfgString) ??
+            throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+         MergeWith(parsedCfg);
+         ConfigFile = null;
+      }
   }
 }
@@ -1,14 +1,21 @@
 namespace bts.websocket_core {
+   using BCG;
+
   using bts.api;
   using bts.api.orderman;
   using bts.md;

+   using Bts.DataStream;
+   using Bts.DataStreamCore;
+   using Bts.DataStreamExpressions;
+
   using mbts.spark;

   using Newtonsoft.Json.Linq;

   using Spark.Api.MarketData;
   using Spark.Api.Orders;
+   using Spark.Core.Collections;

   using System;
   using System.Diagnostics.CodeAnalysis;
@@ -22,6 +29,7 @@ namespace bts.websocket_core {
         PRICE_STATS,
         PRICE_TICKERS,
         FILLS,
+         DATA_STREAM,
         UNKNOWN = 0
      }
      public bool Valid { get; set; } = true;
@@ -91,6 +99,47 @@ namespace bts.websocket_core {
               hdlr.SendMessage(WebsocketJsonParsers.to_json<OrderRecordTupleWithRoot, uint>(upd)));
         };
      }
+
+      private static Action<WebsocketSubscriptionHandler> GetDataStreamSubscriber(ViewStream vs, string subId) {
+         return Subscribe;
+
+         void Subscribe(WebsocketSubscriptionHandler hdlr) {
+            // Declare dictionaries here to reduce allocations in the callback. Use a lock
+            // to be safe (callbacks are likely single-threaded, this is just insurance).
+            var syncRoot = new object();
+            var removedRows = new Dictionary<int, object>();
+            var changedRows = new Dictionary<int, object>();
+            var addedRows = new Dictionary<int, object>();
+
+            hdlr.dis = vs.AsObservable().Subscribe(OnData);
+
+            void OnData(DataPacket dp) {
+               lock (syncRoot) {
+                  removedRows.Clear();
+                  changedRows.Clear();
+                  addedRows.Clear();
+                  foreach (IBlueRow r in dp.deletedRows) {
+                     var id = r.Id();
+                     removedRows[id] = r;
+                  }
+                  foreach (IBlueRow r in dp.addedRows) {
+                     var id = r.Id();
+                     if (removedRows.Remove(id)) {
+                        changedRows.Add(id, r);
+                     } else {
+                        addedRows.Add(id, r);
+                     }
+                  }
+                  hdlr.SendMessage(new {
+                     added = addedRows.Values,
+                     changed = changedRows.Values,
+                     removed = removedRows.Values,
+                  });
+               }
+            }
+         }
+      }
+
      private static Action<WebsocketSubscriptionHandler> GetFillsSubscriber(FillsSubscriptionFilter cri) {
         return (WebsocketSubscriptionHandler hdlr) => {
            hdlr.dis = hdlr._OrdersClient.fills(cri).Subscribe(
@@ -149,6 +198,39 @@ namespace bts.websocket_core {
         return true;
      }

+      static bool TryGetDataStream(
+         WebsocketRequest req,
+         [NotNullWhen(true)] out ViewStream? stream,
+         [NotNullWhen(false)] out string error
+      ) {
+         stream = null;
+         error = "";
+         var query = (req.parameters?["query"] as JValue)?.Value<string>();
+         if (query is null) {
+            error = "'params.query' field not found (expected a string)";
+            return false;
+         }
+         var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+         if (exp is null) {
+            error = "'params.query' expression has invalid syntax";
+            return false;
+         }
+         Exception? exception = null;
+         try {
+            stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+         } catch (Exception ex) {
+            exception = ex;
+         }
+         if (stream is null) {
+            error = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+            if (exception is not null) {
+               error += ". " + exception.Message;
+            }
+            return false;
+         }
+         return true;
+      }
+
      static bool TryGetFilter(WebsocketRequest req, [NotNullWhen(true)] out SecurityCriteria? cri, out string message) {
         cri = null;
         if (req.parameters?["filter"] is not JObject filter) {
@@ -183,10 +265,13 @@ namespace bts.websocket_core {
      }

      public static bool TryCreate(
-         WebsocketRequest req, MarketDataClient md, OrdersClient oc,
+         WebsocketRequest req,
+         MarketDataClient md,
+         OrdersClient oc,
         Action<Object> sm,
         [NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr,
-         out string status) {
+         out string status
+      ) {
         hdlr = null;

         if (
@@ -247,6 +332,14 @@ namespace bts.websocket_core {
                  sub = GetOrdersSubscriber(filter);
                  break;
               }
+            case SUBSCRIPTION_TYPE.DATA_STREAM: {
+                  // TODO: make this nullable
+                  if (!TryGetDataStream(req, out var filter, out status)) {
+                     return false;
+                  }
+                  sub = GetDataStreamSubscriber(filter, id.Value.Val);
+                  break;
+               }
            default: {
                  status = "Incorrect subscription type";
                  return false;
@@ -19,13 +19,15 @@
      <PrivateAssets>all</PrivateAssets>
    </PackageReference>
    <PackageReference Include="System.Net.WebSockets" Version="4.3.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
    <PackageReference Include="Microsoft.AspNetCore.Cors" Version="2.2.0" />
  </ItemGroup>

  <ItemGroup>
    <ProjectReference Include="..\spark_core\spark_core.csproj" />
    <ProjectReference Include="..\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\DataStreamSuite\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\..\DataStreamSuite\SharedStreams\SharedStreams.csproj" />
  </ItemGroup>

  <ItemGroup>
@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 3.5)
 project(libspark)
 set(CMAKE_SHARED_LIBRARY_PREFIX "")
+include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/GenerateDefaultConfig.cmake)
@@ -71,9 +72,7 @@ if (DEFINED DOTNET_WINDOWS_BINARY_DIR)
    "Copy ${PROJECT_NAME} bins to Dotnet Windows build dir")
 endif()

-if(BUILD_SPARKD)
-  install(TARGETS ${PROJECT_NAME} DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
-endif()
+install(TARGETS ${PROJECT_NAME} DESTINATION "${CMAKE_INSTALL_BINDIR}")

 if(WIN32)
  include(${CMAKE_SOURCE_DIR}/cmake/vs_filters.cmake)
@@ -27,4 +27,5 @@ enum Venue {
  VENUE_OKX = 15;
  VENUE_TAIFEX = 16;
  VENUE_CBOE = 17;
+  VENUE_TWSE = 18;
 }
@@ -14,4 +14,5 @@ message DatabentoMarketDataConfig {
  string key = 60;
  string dataset = 70;
  bool filter_securities_one_year= 80;
+  bool load_reference_data = 90;
 }
@@ -12,6 +12,7 @@ import "lib/api/md/EmumbaDeribitMarketDataConfig.proto";
 import "lib/api/md/OkxMarketDataConfig.proto";
 import "lib/api/md/GlbxMarketDataConfig.proto";
 import "lib/api/md/TaifexMarketDataConfig.proto";
+import "lib/api/md/TwseMarketDataConfig.proto";
 import "lib/api/md/CboeMarketDataConfig.proto";

 option csharp_namespace = "bts.api.md";
@@ -29,6 +30,7 @@ message MarketDataConfig {
  repeated GlbxMarketDataConfig glbx = 8;
  repeated CboeMarketDataConfig cboe = 9;
  repeated TaifexMarketDataConfig taifex =10;
+  repeated TwseMarketDataConfig twse =11;
  // Add an entry here for all the exchanges that need their market data to be recorded

  repeated EmumbaDeribitMarketDataConfig emumba_deribit = 100;
@@ -23,6 +23,8 @@ std::set<bts::api::Venue> MarketDataConfigUtils::get_live_exchanges(
      rv.insert(bts::api::VENUE_GLBX);
   if (cfg.taifex_size()>0)
      rv.insert(bts::api::VENUE_TAIFEX);
+   if (cfg.twse_size()>0)
+      rv.insert(bts::api::VENUE_TWSE);
   if (cfg.cboe_size()>0)
      rv.insert(bts::api::VENUE_CBOE);
   return rv;
@@ -9,7 +9,9 @@ option cc_enable_arenas = true;
 // Config to control the recording of the market data for an exchange.
 // This config is part of MarketDataConfig.proto
 message MarketDataRecordingConfig {
-   uint32 num_of_seconds_to_run = 10;
+   uint32 num_of_seconds_to_run_file_writer=12;
+   uint32 num_of_seconds_to_run_db_writer=13;
   bts.api.Venue venue = 20;
   uint32 n_book_levels = 30;
+   uint32 batch_size = 40;
 }
@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+import "lib/api/domain/SecurityCriteria.proto";
+
+package bts.api.md;
+
+option csharp_namespace = "bts.api.md";
+message TwseConnectionDetails{
+  string live_data_ip = 30;
+}
+
+message TwseMarketDataConfig {
+  uint32 unique_check_value = 1;
+  bool read_from_file = 20;
+  string realtime_data_file_name = 22;
+  TwseConnectionDetails connection_details = 30;
+  bts.api.domain.SecurityCriteria security_criteria = 3;
+  
+}
@@ -72,6 +72,7 @@ bool AdminServiceUtils::has_exchange(
   case api::Venue::VENUE_OKX:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
   case api::Venue::VENUE_TAIFEX:
   case api::Venue::VENUE_CBOECFE:
   case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -156,6 +157,8 @@ AdminServiceUtils::resolve_connection(
   const bts::api::admin::ExchangeConnection *rv = nullptr;
   switch (sec.exchange()) {
   case api::Venue::VENUE_GLBX: {
+      //CME EXECUTION NEEDS TO BE UPDATED, probably merge with VENUE_BINANCE_USDS_FUTURES
+      
      auto deployment_type =
          boost::to_lower_copy(config::get<config::DeploymentType>().type());
      rv = resolve_cme_connection(data, account, sec);
@@ -185,6 +188,7 @@ AdminServiceUtils::resolve_connection(
   case api::Venue::VENUE_BINANCE:
   case api::Venue::VENUE_BINANCE_US:
   case api::Venue::VENUE_COINBASE:
+   case api::Venue::VENUE_TWSE:
   case api::Venue::VENUE_TAIFEX:
   case api::Venue::VENUE_CBOE:
   case api::Venue::VENUE_BINANCE_USDS_FUTURES: {
@@ -227,6 +231,7 @@ void AdminServiceUtils::fill_connections_from_config(
      connection.set_name(c.name());

      switch (c.venue()) {
+      //CME EXECUTION NEEDS TO BE UPDATED, probably merge default case
      case bts::api::VENUE_GLBX:
         if (c.cme_market_segment_id() == 0)
            connection.mutable_cme_connection()->mutable_cgw_connection();
@@ -236,10 +241,7 @@ void AdminServiceUtils::fill_connections_from_config(
                ->set_market_segment_id(c.cme_market_segment_id());
         add_connection(admin_data, connection);
         break;
-      case bts::api::VENUE_CBOE:
-         break;
-         case bts::api::VENUE_TAIFEX:
-         break;
+
      case bts::api::VENUE_SIM:
      case bts::api::VENUE_SIMICE:
         connection.mutable_sim_connection();
@@ -250,6 +252,8 @@ void AdminServiceUtils::fill_connections_from_config(
             c.sender_compid());
         add_connection(admin_data, connection);
         break;
+      case bts::api::VENUE_TAIFEX:
+      case bts::api::VENUE_CBOE:
      default:
         connection.mutable_general_connection()->set_exchange(c.venue());
         add_connection(admin_data, connection);
@@ -458,6 +462,7 @@ std::string AdminServiceUtils::find_account_exchange_account(
   case api::Venue::VENUE_OKX:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
   case api::Venue::VENUE_TAIFEX:
   case api::Venue::VENUE_CBOECFE:
   case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -571,6 +576,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
      return rv;
   }
   switch (sec.exchange()) {
+   case api::Venue::VENUE_CBOE:{//subaccount not added
+       rv.set_session("cboe_1");
+      rv.set_account(account_ptr->name());
+      rv.set_operator_id(user_ptr->name());
+      out_result.set_is_success(true);
+      return rv;
+   }
   case api::Venue::VENUE_DERIBIT: {
      rv.set_session("deribit_1");
      rv.set_account(account_ptr->name());
@@ -733,6 +745,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
      rv.set_customer_or_firm(sim_account_data->origin());
      return rv;
   } break;
+   case api::Venue::VENUE_TAIFEX:{
+       rv.set_session("taifex_1");
+      rv.set_account(account_ptr->name());
+      rv.set_operator_id(user_ptr->name());
+      out_result.set_is_success(true);
+      return rv;
+   }
   case api::VENUE_FTX:     // TODO
   case api::VENUE_BINANCE: // TODO
   case api::VENUE_BINANCE_US:
@@ -741,8 +760,7 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
   case api::VENUE_OKX:
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_BTS:
-   case api::Venue::VENUE_CBOE:
-   case api::Venue::VENUE_TAIFEX:
+   case api::Venue::VENUE_TWSE:
   case api::Venue::VENUE_CBOECFE:
   case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
   case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
@@ -1138,6 +1156,7 @@ std::string AdminServiceUtils::find_user_exchange_id(
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
   case api::Venue::VENUE_TAIFEX:
   case api::Venue::VENUE_CBOECFE:
   case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -1176,6 +1195,7 @@ std::string AdminServiceUtils::find_user_company_id(
   case api::Venue::VENUE_UNSET:
   case api::Venue::VENUE_BTS:
   case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
   case api::Venue::VENUE_TAIFEX:

   case api::Venue::VENUE_CBOECFE:
@@ -0,0 +1,22 @@
+MarexFixCfg {
+    value{
+        customer_or_firm :0
+        handl_inst: 2
+        rule80a: 1
+        heart_bt_int : 30
+        counterparty_port :22754
+        name: "trafix"
+        msg_type: "Order_Single"
+        protocol_version: "FIX_42"
+        sender_sub_id: "BTS"
+        target_sub_id: "TRFXMRX"
+        client_id: "BTS"
+        counterparty_host : "74.217.55.230"
+        open_close :"O"
+        max_retries: 500
+        retry_interval: 600
+        ex_destination : "W"
+        security_exchange : "CBOE"
+        currency : "USD"
+    }
+}
@@ -19,6 +19,14 @@ SymbolLists {
      symbol: "BTC"
      symbol: "ETH"
    }
+    list {
+      key: "TWSE"
+      symbol: "2330"
+      symbol: "2317"
+      symbol: "2454"
+      symbol: "2603"
+      symbol: "2615"
+    }
    list {
      key: "TAIFEX"
      symbol: "QFFH"
@@ -59,6 +67,11 @@ Symbols {
      future: @SymbolLists.list[GLBX].symbol
      spread: @SymbolLists.list[GLBX].symbol
    }
+    venues{
+      name: "VENUE_TWSE"
+      future: @SymbolLists[TWSE].symbol
+      option: @SymbolLists[TWSE].symbol
+    }
    venues{
      name: "VENUE_TAIFEX"
      future: @SymbolLists[TAIFEX].symbol
@@ -91,6 +104,11 @@ Symbols {
      future: @SymbolLists.list[ICE].symbol
      spread: @SymbolLists.list[ICE].symbol
    }
+    venues{
+      name: "VENUE_TWSE"
+      future: @SymbolLists[TWSE].symbol
+      option: @SymbolLists[TWSE].symbol
+    }
    venues{
      name: "VENUE_TAIFEX"
      future: @SymbolLists[TAIFEX].symbol
@@ -125,6 +143,11 @@ Symbols {
      option: @SymbolLists.list[DERIBIT].symbol
      spread: @SymbolLists.list[DERIBIT].symbol
    }
+    venues{
+      name: "VENUE_TWSE"
+      future: @SymbolLists.list[TWSE].symbol
+      option: @SymbolLists.list[TWSE].symbol
+    }
    venues{
      name: "VENUE_TAIFEX"
      future: @SymbolLists.list[TAIFEX].symbol
@@ -6,8 +6,8 @@ tickConfig
 		tick_entry {key:"TXF",value:1}, 
        tick_entry {key:"MXF",value:1}, 
        tick_entry {key:"TMF",value:1}, 
-        tick_entry {key:"CDF",value:1},
-        tick_entry {key:"QFF",value:1},
+        tick_entry {key:"CDF",value:5},
+        tick_entry {key:"QFF",value:5},
 	}
 }

@@ -0,0 +1,33 @@
+syntax = "proto3";
+package bts.config;
+message MarexFixCfg {
+    enum HandlInst {
+        UNDEFINED = 0;
+        DMA = 1;
+        ALGO = 2;
+        CARE = 3;
+    }
+    enum Rule80A {
+        UNUSED = 0;
+        A = 1;
+        P = 2;
+    }
+    string msg_type = 1;
+    string protocol_version = 2;
+    HandlInst handl_inst = 3;
+    string sender_sub_id = 4;
+    string target_sub_id = 5;
+    string client_id = 6;
+    Rule80A rule80a = 7;
+    string counterparty_host = 8;
+    int32 heart_bt_int = 9;
+    int32 counterparty_port = 10;
+    uint32 customer_or_firm =11;
+    string open_close =12;
+    string name = 13;
+    int32 max_retries = 14;
+    uint32 retry_interval = 15;
+    string ex_destination = 16;
+    string security_exchange = 17;
+    string currency = 18;
+}
@@ -50,7 +50,11 @@ class TimeSpan {
 constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
 constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr TimeSpan operator-(TimeSpan lhs, TimeSpan rhs) noexcept;
+constexpr TimeSpan operator+(TimeSpan lhs, TimeSpan rhs) noexcept;

 std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;

@@ -116,10 +120,26 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
   return lhs.nanos() < rhs.nanos();
 }

+constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() <= rhs.nanos();
+}
+
 constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
   return lhs.nanos() > rhs.nanos();
 }

+constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return lhs.nanos() >= rhs.nanos();
+}
+
+constexpr TimeSpan operator-(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return TimeSpan{lhs.nanos() - rhs.nanos()};
+}
+
+constexpr TimeSpan operator+(TimeSpan lhs, TimeSpan rhs) noexcept {
+   return TimeSpan{lhs.nanos() + rhs.nanos()};
+}
+
 } // namespace bts::core

 FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());
@@ -25,6 +25,8 @@ api::domain::Venue DomainSecurityUtils::convert(bts::api::Venue venue) {
      return DOMAIN_VENUE(DERIBIT);
   case bts::api::Venue::VENUE_CBOE:
      return DOMAIN_VENUE(CBOE);
+   case bts::api::Venue::VENUE_TWSE:
+      return DOMAIN_VENUE(TWSE);
   case bts::api::Venue::VENUE_TAIFEX:
      return DOMAIN_VENUE(TAIFEX);
   case bts::api::Venue::VENUE_BTS:
@@ -59,6 +61,8 @@ bts::api::Venue DomainSecurityUtils::convert(api::domain::Venue venue) {
      return api::Venue::VENUE_DERIBIT;
   case DOMAIN_VENUE(CBOE):
      return api::Venue::VENUE_CBOE;
+   case DOMAIN_VENUE(TWSE):
+      return api::Venue::VENUE_TWSE;
   case DOMAIN_VENUE(TAIFEX):
      return api::Venue::VENUE_TAIFEX;      
   case DOMAIN_VENUE(FTX):
@@ -171,6 +175,9 @@ bool DomainSecurityUtils::get_security_desc(
   case bts::api::Security::ExchangeDataCase::kCboe:
      desc = security.cboe().security_desc();
      break;
+   case bts::api::Security::ExchangeDataCase::kTwse:
+      desc = security.twse().security_desc();
+      break;
   case bts::api::Security::ExchangeDataCase::kTaifex:
      desc = security.taifex().security_desc();
      break;
@@ -3,6 +3,7 @@

 #include "lib/config/config.h"
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
+#include "lib/md/common.h"
 #include "lib/utils/datetime.h"

 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -11,51 +12,15 @@
 namespace bts {
 namespace securities {

-std::string to_str(api::Security const &sec) {
-   std::string serialized;
-   google::protobuf::TextFormat::PrintToString(sec, &serialized);
-   return serialized;
-}
-
-std::pair<bool, api::Security> from_str(std::string const &str) {
-   api::Security result;
-   auto success = google::protobuf::TextFormat::ParseFromString(str, &result);
-   return std::make_pair(success, result);
-}
-
-std::string descriptor(api::Security const &sec) {
-   switch (sec.ExchangeData_case()) {
-   case api::Security::kTaifex:
-      return sec.taifex().security_desc();
-   case api::Security::kGlbx:
-      return sec.glbx().security_desc();
-   case api::Security::kCboe:
-      return sec.cboe().security_desc();
-   case api::Security::kCfe:
-      return sec.cfe().security_desc();
-   case api::Security::kIce:
-      return sec.ice().security_desc();
-   case api::Security::kDeribit:
-      return sec.deribit().security_desc();
-   case api::Security::kFtx:
-      return sec.ftx().security_desc();
-   case api::Security::kBinance:
-      return sec.binance().security_desc();
-   case api::Security::kBinanceUsdsFutures:
-      return sec.binance_usds_futures().security_desc();
-   case api::Security::kBinanceUs:
-      return sec.binance_us().security_desc();
-   case api::Security::kCoinbase:
-      return sec.coinbase().security_desc();
-   case api::Security::kOkx:
-      return sec.okx().security_desc();
-   default:
-      return "No descriptor for non glbx / ice exchange";
-   }
-}
+using bts::api::Security;
+using bts::api::Venue;

 api::SecurityType security_type(api::Security const &security) {
-   switch (security.Contract_case()) {
+   return security_type(security.Contract_case());
+}
+
+api::SecurityType security_type(api::Security::ContractCase contract_case) {
+   switch (contract_case) {
   case api::Security::ContractCase::kFuture:
      return api::SecurityType::SECURITYTYPE_FUTURE;
   case api::Security::ContractCase::kOption:
@@ -70,16 +35,6 @@ api::SecurityType security_type(api::Security const &security) {
      return api::SecurityType::SECURITYTYPE_UNSET;
   }
   return api::SecurityType::SECURITYTYPE_UNSET;
-   // if (sec.has_future())
-   //    return api::SecurityType::SECURITYTYPE_FUTURE;
-   // else if (sec.has_option())
-   //    return api::SecurityType::SECURITYTYPE_OPTION;
-   // else if (sec.has_spread())
-   //    return api::SecurityType::SECURITYTYPE_SPREAD;
-   // else if (sec.has_synthetic())
-   //    return api::SecurityType::SECURITYTYPE_SYNTHETIC;
-   // else
-   //    return api::SecurityType::SECURITYTYPE_UNSET;
 }

 boost::gregorian::date smallest_leg(
@@ -140,5 +95,91 @@ bool is_expired(api::Security const &security) {
   return is_expired(security, datetime::local_now());
 }

+void write_date_string(
+    uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) {
+   result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2)
+          << std::setfill('0') << day << "/" << year - 2000;
+}
+
+std::ostringstream &write_leg_descriptor(
+    std::ostringstream &result, char separator, const api::ContractLeg &leg) {
+   bts_assert(leg.security().id() != md::invalid_security_key);
+   // Keep the string small, yet readable for diagnostic purposes.
+   result << (leg.side() == api::SIDE_BUY ? '+' : '-') << separator
+          << leg.quantity() << "*(i=" << leg.security().id()
+          << ",m=" << leg.price_multiplier() << ",d=" << leg.leg_delta() << ")";
+   return result;
+}
+
+std::ostringstream &write_future_descriptor(
+    std::ostringstream &result, const api::FutureContract &future) {
+   write_date_string(
+       future.expiration_year(), future.expiration_month(),
+       future.expiration_day(), result);
+   return result;
+}
+
+std::ostringstream &write_option_descriptor(
+    std::ostringstream &result,
+    char separator,
+    const api::OptionContract &option) {
+   write_date_string(
+       option.expiration_year(), option.expiration_month(),
+       option.expiration_day(), result);
+   result << separator << option.strike()
+          << (option.call_put() == api::PUTORCALL_CALL ? 'C' : 'P');
+   return result;
+}
+
+std::ostringstream &write_legs_descriptor(
+    std::ostringstream &result,
+    char separator,
+    const google::protobuf::RepeatedPtrField<api::ContractLeg> &legs) {
+   bts_assert(legs.size() > 0);
+   auto it = legs.begin();
+   write_leg_descriptor(result, separator, *it);
+   while (++it != legs.end()) {
+      result << separator;
+      write_leg_descriptor(result, separator, *it);
+   }
+   return result;
+}
+
+std::string unique_descriptor(const api::Security &security) {
+   const char separator = ' ';
+   std::ostringstream result;
+   result << api::Venue_Name(security.exchange()).substr(6) << separator
+          << api::SecurityType_Name(security_type(security)).substr(12)
+          << descriptor(security);
+   switch (security.Contract_case()) {
+   case Security::ContractCase::kOption:
+      result << separator;
+      write_option_descriptor(result, separator, security.option());
+      break;
+   case Security::ContractCase::kFuture:
+      result << separator;
+      write_future_descriptor(result, security.future());
+      break;
+   case Security::ContractCase::kSpot:
+      break;
+   case Security::ContractCase::kSpread:
+      result << separator;
+      write_legs_descriptor(result, separator, security.spread().legs());
+      break;
+   case Security::ContractCase::kSynthetic: {
+      result << "pc="
+             << api::SyntheticContract::PriceCalc_Name(
+                    security.synthetic().price_calc());
+      write_legs_descriptor(result, separator, security.synthetic().legs());
+      break;
+   }
+   default:
+      bts_assert_msg(
+          false, "unique_descriptor failed due to invalid contract case");
+      break;
+   }
+   return result.str();
+}
+
 } // namespace securities
 } // namespace bts
@@ -2,6 +2,7 @@

 #include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/SecurityType.h"
 #include "lib/domain_model/Side.h"
 #include "lib/domain_model/Venue.h"
 #include "lib/domain_model/common.h"
@@ -25,11 +26,27 @@ class ICEExchangeData;
 } // namespace api

 namespace securities {
-SecurityKey security_key(api::Security const &);
-std::string to_str(api::Security const &);
-std::pair<bool, api::Security> from_str(std::string const &);
-std::string descriptor(api::Security const &);
+
+/**
+ * Gets a descriptor string for a security (should be unique within a particular exchange).
+ */
+const std::string &descriptor(
+    const api::Security &sec, const std::string &default_rv = "UNDEFINED");
+
+/**
+ * Gets a descriptor string for a security that is unique within the entire
+ * system (not just exchange).
+ */
+std::string unique_descriptor(const api::Security &security);
+
 api::SecurityType security_type(api::Security const &);
+api::SecurityType security_type(api::Security::ContractCase contract_case);
+
+void write_date_string(
+    std::uint32_t year,
+    std::uint32_t month,
+    std::uint32_t day,
+    std::ostringstream &result);

 inline std::string const &side_name(api::Side side) {
   return api::Side_Name(side);
@@ -234,6 +251,43 @@ inline Price tick_size(api::Security const &sec) {
   return Price(sec.contract_spec().low_tick());
 }

+inline const std::string &descriptor(
+    const api::Security &sec, const std::string &default_rv) {
+   switch (sec.ExchangeData_case()) {
+   case api::Security::kGlbx:
+      return sec.glbx().security_desc();
+   case api::Security::kCboe:
+      return sec.cboe().security_desc();
+   case api::Security::kCfe:
+      return sec.cfe().security_desc();
+   case api::Security::kIce:
+      return sec.ice().security_desc();
+   case api::Security::kDeribit:
+      return sec.deribit().security_desc();
+   case api::Security::kFtx:
+      return sec.ftx().security_desc();
+   case api::Security::kBinance:
+      return sec.binance().security_desc();
+   case api::Security::kBinanceUsdsFutures:
+      return sec.binance_usds_futures().security_desc();
+   case api::Security::kBinanceUs:
+      return sec.binance_us().security_desc();
+   case api::Security::kCoinbase:
+      return sec.coinbase().security_desc();
+   case api::Security::kOkx:
+      return sec.okx().security_desc();
+   case api::Security::kTwse:
+      return sec.twse().security_desc();
+   case api::Security::kTaifex:
+      return sec.taifex().security_desc();
+   default: {
+      if (sec.has_synthetic()) {
+         return sec.synthetic().name();
+      }
+      return default_rv;
+   }
+   }
+}
 // Looking to round md::IntPrice? Look in IntPrice.h

 // computes price_2 - price_1 in ticks
@@ -314,3 +368,5 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) {

 } // namespace securities
 } // namespace bts
+
+FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s));
@@ -20,6 +20,7 @@ enum Venue {
     VENUE_COINBASE = 450;
     VENUE_OKX = 460;
     VENUE_TAIFEX = 500;
+     VENUE_TWSE = 600;
 }

 enum SecurityType {
@@ -87,6 +88,11 @@ message CBOEExchangeData {
    int32 class_key = 30;
    string source = 40;
 }
+message TWSEExchangeData {
+    int64 exchange_id = 10;
+    int32 class_key = 30;
+    string security_desc = 20;
+}
 message TAIFEXExchangeData {
    int64 exchange_id = 10;
    int32 class_key = 30;
@@ -233,6 +239,7 @@ message Security {
       CoinbaseExchangeData coinbase = 290;
       OkxExchangeData okx = 300;
       TAIFEXExchangeData taifex = 400;
+       TWSEExchangeData twse = 500;
   }
 }
 message SecurityList {
@@ -0,0 +1,30 @@
+#pragma once
+
+#include "lib/domain_model/Security.pb.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+PB_ENUM_FROM_JSON(SecurityType)
+PB_ENUM_TO_JSON(SecurityType)
+
+std::ostream &operator<<(std::ostream &out, const SecurityType &type) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline std::ostream &operator<<(
+    std::ostream &out, const SecurityType &type) noexcept {
+   out << SecurityType_Name(type);
+   return out;
+}
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::SecurityType, t, "{}", bts::api::SecurityType_Name(t));
@@ -13,6 +13,17 @@ namespace bts::api {
 PB_ENUM_FROM_JSON(Venue)
 PB_ENUM_TO_JSON(Venue)

+std::ostream &operator<<(std::ostream &out, Venue venue) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline std::ostream &operator<<(std::ostream &out, Venue venue) noexcept {
+   out << Venue_Name(venue);
+   return out;
+}
+
 } // namespace bts::api

 FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v));
@@ -0,0 +1,383 @@
+#include "websocket_client.h"
+
+#include "lib/core/TimeSpan.h"
+#include "lib/core/TimeStamp.h"
+#include "lib/io/websocket/websocket_client_state.h"
+#include "lib/io/websocket/websocket_close_status.h"
+#include "lib/io/websocket/websocket_error_code.h"
+#include "lib/io/websocket/websocketpp_exception_formatter.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/formatters/exception_formatter.h"
+
+#include <atomic>
+#include <exception>
+#include <fmt/format.h>
+#include <mutex>
+#include <nlohmann/json.hpp>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <websocketpp/close.hpp>
+#include <websocketpp/common/connection_hdl.hpp>
+#include <websocketpp/common/memory.hpp>
+#include <websocketpp/common/system_error.hpp>
+#include <websocketpp/common/thread.hpp>
+#include <websocketpp/config/asio_client.hpp>
+#include <websocketpp/config/core_client.hpp>
+#include <websocketpp/roles/client_endpoint.hpp>
+
+namespace bts::websocket {
+
+struct WebsocketClientSnapshot {
+   int client_id;
+   WebsocketClientState state;
+   int version;
+};
+
+} // namespace bts::websocket
+
+FMT_IMPL(
+    bts::websocket::WebsocketClientSnapshot,
+    t,
+    "id: {}; state: {}; version: {}",
+    t.client_id,
+    t.state,
+    t.version);
+
+namespace bts::websocket {
+
+using namespace std::chrono_literals;
+
+using bts::core::TimeSpan;
+using bts::core::TimeStamp;
+using nlohmann::json;
+using std::lock_guard;
+using std::map;
+using std::mutex;
+using std::optional;
+using std::size_t;
+using std::string;
+using std::thread;
+using std::uint32_t;
+using websocketpp::config::asio_tls_client;
+
+using WebsocketppClient =
+    websocketpp::client<websocketpp::config::asio_tls_client>;
+using ConnectionHdl = websocketpp::connection_hdl;
+using ConnectionPtr = WebsocketppClient::connection_ptr;
+using MessageTypePtr = websocketpp::config::core_client::message_type::ptr;
+using SslContext = boost::asio::ssl::context;
+using State = WebsocketClientState;
+
+// ws endpoint http://markets.api.uat.cmegroup.com/marketdatastream/v1
+
+#define _ASSERT_MSGF(condition, ...)                                           \
+   bts_assert_msgf(condition, "WebsocketClient::" __VA_ARGS__)
+
+#define DIAG(...) LOG(Gen, Diag, __VA_ARGS__)
+#define DIAG_FN(...) LOG(Gen, Diag, "WebsocketClient::" __VA_ARGS__)
+#define INFO(...) LOG(Gen, Info, __VA_ARGS__)
+#define INFO_FN(...) LOG(Gen, Info, "WebsocketClient::" __VA_ARGS__)
+#define WARN(...) LOG(Gen, Warn, __VA_ARGS__)
+#define WARN_FN(...) LOG(Gen, Warn, "WebsocketClient::" __VA_ARGS__)
+#define ERROR(...) LOG(Gen, Error, __VA_ARGS__)
+#define ERROR_FN(...) LOG(Gen, Error, "WebsocketClient::" __VA_ARGS__)
+
+struct WebsocketClient::PImpl {
+   string url;
+   WebsocketppClient client;
+   uint32_t timeout_secs;
+   WebsocketppClient::connection_ptr con;
+   optional<MessageHandler> message_handler;
+   size_t sub_id_counter{0};
+   string server_name;
+   optional<thread> io_thread;
+   int client_id{client_id_counter++};
+   mutex instance_mx;
+   State state;
+   int version = 0;
+
+   ~PImpl() {
+      if (io_thread) {
+         io_thread->join();
+      }
+   }
+
+   int set_state(State state) {
+      if (this->state == state) {
+         return version;
+      }
+      version++;
+      INFO(
+          "State changing from {} -> {}. id: {}; version: {}", this->state,
+          state, client_id, version);
+      this->state = state;
+      return version;
+   }
+
+   constexpr WebsocketClientSnapshot snapshot() const {
+      return {client_id, state, version};
+   }
+
+   void close(websocketpp::close::status::value code, std::string reason) {
+      lock_guard<mutex> lock{instance_mx};
+      INFO_FN(
+          "close. {}; code: {}; reason: {}", snapshot(),
+          WebsocketCloseStatus{code}, reason);
+      switch (state.value) {
+      case State::OPENING:
+         WARN_FN(
+             "close. Cannot close while websocket is still opening. {}",
+             snapshot());
+         // Wait until we get the 'open' callback, then initiate the close
+         set_state(State::WAITING_TO_CLOSE);
+         return;
+      case State::OPEN:
+         set_state(State::CLOSING);
+         break;
+      default:
+         return;
+      }
+      websocketpp::lib::error_code ec;
+      client.close(con->get_handle(), code, reason, ec);
+      if (ec) {
+         ERROR_FN(
+             "close. {}; code: {}; reason: {}; error: {}", snapshot(),
+             WebsocketCloseStatus{code}, reason, ec);
+      }
+   }
+
+   websocketpp::lib::error_code send_text(const string &text) {
+      INFO_FN("send_text. {}; text: {}", snapshot(), text);
+      websocketpp::lib::error_code ec;
+      client.send(con, text, websocketpp::frame::opcode::value::TEXT, ec);
+      if (ec) {
+         ERROR_FN("send_text. {}; text: {}; err: {}", snapshot(), text, ec);
+      }
+      return ec;
+   }
+
+   /**
+    * Connects to the websocket server
+    */
+   void connect(const map<string, string> &headers) {
+      connect_in_background(headers);
+      // Wait for connect
+      if (timeout_secs > 0) {
+         wait_for_connect(TimeSpan::from_seconds(timeout_secs));
+      }
+   }
+
+   /**
+    * Connects to the websocket server in the background.
+    *
+    * @note Doesn't block while waiting for the connection to be established
+    */
+   void connect_in_background(const map<string, string> &headers) {
+      lock_guard<mutex> lock{instance_mx};
+      INFO_FN("connect_no_timeout. {}", snapshot());
+      switch (state.value) {
+      case State::IDLE:
+         break;
+      default:
+         WARN_FN("connect_no_timeout. Unexpected state. {}", snapshot());
+         return;
+      }
+      set_state(State::OPENING);
+      try {
+         init_logging();
+         init_handlers();
+         // Get connection
+         websocketpp::lib::error_code ec;
+         con = client.get_connection(url, ec);
+         if (ec) {
+            ERROR_FN("connect_no_timeout. {}; err: {}", snapshot(), ec);
+            return;
+         }
+         // Add headers
+         for (const auto &[k, v] : headers) {
+            con->append_header(k, v);
+         }
+         // Connect
+         client.connect(con);
+         bts_assert_msgf(
+             !io_thread, "Websocket io_thread already instantiated. {}",
+             snapshot());
+         io_thread = std::thread{[this]() {
+            INFO("Websocket io thread begin. {}", snapshot());
+            try {
+               client.run();
+            } catch (const websocketpp::exception &ex) {
+               ERROR_FN(
+                   "io_thread. Websocket io threw websocketpp::exception. {}; "
+                   "err: {}",
+                   snapshot(), ex);
+            } catch (const std::exception &ex) {
+               ERROR_FN(
+                   "io_thread. Websocket io threw std::exception. {}; err: {}",
+                   snapshot(), ex);
+            } catch (...) {
+               bts_assert_msgf(
+                   false,
+                   "io_thread. Websocket io threw unknown exception. Could "
+                   "this have been thrown by a callback delegate? {}",
+                   snapshot());
+            }
+            INFO("Websocket 'run' thread end. {}", snapshot());
+         }};
+      } catch (const std::exception &ex) {
+         ERROR_FN(
+             "connect_no_timeout. Failed to connect. {}; err: {}", snapshot(),
+             ex);
+         set_state(State::IDLE);
+         // Don't wait for connect
+         return;
+      }
+   }
+
+   void wait_for_connect(const TimeSpan timeout) {
+      INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout);
+      const auto start = TimeStamp::now();
+      while (state == State::OPENING) {
+         const auto elapsed = TimeStamp::now() - start;
+         const auto rem = timeout - elapsed;
+         if (rem < TimeSpan::zero()) {
+            throw std::runtime_error{fmt::format(
+                "Timed out waiting for connection after {} secs. {}",
+                elapsed.total_seconds(), snapshot())};
+         }
+         WARN_FN(
+             "wait_for_connect. Still waiting. {}; timeout: {}; rem: {}",
+             snapshot(), timeout, rem);
+         std::this_thread::sleep_for(std::chrono::seconds(1));
+      }
+   }
+
+ private:
+   static std::atomic<int> client_id_counter;
+
+   auto on_tls_init(ConnectionHdl /*hdl*/) {
+      INFO_FN("on_tls_init. state: {}; version: {}", state, version);
+      auto result =
+          websocketpp::lib::make_shared<SslContext>(SslContext::sslv23_client);
+      result->set_verify_mode(boost::asio::ssl::verify_none);
+      return result;
+   }
+
+   void init_handlers() {
+      INFO_FN("init_handlers. state: {}; version: {}", state, version);
+      bts_assert(!io_thread);
+      client.init_asio();
+      client.set_message_handler(
+          [this](auto hdl, auto msg) { on_message(hdl, msg); });
+      client.set_open_handler([this](auto hdl) { on_open(hdl); });
+      client.set_tls_init_handler(
+          [this](auto hdl) { return on_tls_init(hdl); });
+      client.set_close_handler([this](auto hdl) { on_close(hdl); });
+      client.set_fail_handler([this](auto hdl) {
+         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
+         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
+      });
+   }
+
+   void init_logging() {
+      INFO_FN("init_logging. {}", snapshot());
+      client.set_access_channels(websocketpp::log::alevel::all);
+      client.clear_access_channels(websocketpp::log::alevel::frame_payload);
+      client.set_error_channels(websocketpp::log::elevel::all);
+   }
+
+   void on_open(ConnectionHdl hdl) {
+      lock_guard<mutex> lock{instance_mx};
+      INFO_FN("on_open. {}", snapshot());
+      const auto ec = client.get_con_from_hdl(hdl)->get_ec();
+      if (ec) {
+         ERROR_FN("on_open. {}; err: {}", snapshot(), ec.message());
+      }
+      switch (state.value) {
+      case State::OPENING:
+         break;
+      default:
+         WARN_FN("on_open. Unexpected state. {}", snapshot());
+         return;
+      }
+      set_state(State::OPEN);
+   }
+
+   void on_message(ConnectionHdl /*hdl*/, MessageTypePtr msg) {
+      DIAG("on_message. {}", snapshot());
+      const auto &payload = msg->get_payload();
+      if (message_handler) {
+         (*message_handler)(payload);
+         return;
+      }
+      ERROR_FN(
+          "on_message. Message handler is null. {}; payload: \n{}", snapshot(),
+          payload);
+   }
+
+   void on_close(ConnectionHdl hdl) {
+      lock_guard<mutex> lock{instance_mx};
+      INFO_FN("on_close. {}", snapshot());
+      switch (state.value) {
+      case State::CLOSING:
+         break;
+      default:
+         WARN_FN("on_close. Unexpected state. {}", snapshot());
+         break;
+      }
+      set_state(State::IDLE);
+      const auto con = client.get_con_from_hdl(hdl);
+      const auto ec = con->get_ec();
+      if (ec) {
+         ERROR_FN(
+             "on_close. {}; local_code: {}; "
+             "local_reason: {}; remote_code: {}; remote_reason: {}; err: {}",
+             snapshot(), con->get_local_close_code(),
+             con->get_local_close_reason(), con->get_remote_close_code(),
+             con->get_remote_close_reason(), ec.message());
+      }
+   }
+};
+
+WebsocketClient::WebsocketClient(std::string url, std::uint32_t timeout_secs)
+    : me_(std::make_unique<PImpl>()) {
+   me_->url = std::move(url);
+   me_->timeout_secs = timeout_secs;
+}
+
+WebsocketClient::~WebsocketClient() {
+   me_->close(websocketpp::close::status::going_away, "");
+}
+
+std::atomic<int> WebsocketClient::PImpl::client_id_counter = 0;
+
+WebsocketClient::MessageHandler &WebsocketClient::message_handler() {
+   if (!me_->message_handler) {
+      me_->message_handler.emplace();
+   }
+   return *me_->message_handler;
+}
+
+void WebsocketClient::connect(
+    const std::map<std::string, std::string> &headers) {
+   me_->connect(headers);
+}
+
+websocketpp::lib::error_code WebsocketClient::send_text(const string &text) {
+   return me_->send_text(text);
+}
+
+void WebsocketClient::disconnect() {
+   me_->close(
+       websocketpp::close::status::going_away,
+       "WebsocketClient::disconnect was called");
+}
+
+bool WebsocketClient::is_connected() const {
+   lock_guard<mutex> lock{me_->instance_mx};
+   return me_->state == State::OPEN;
+}
+
+} // namespace bts::websocket
@@ -0,0 +1,32 @@
+#pragma once
+
+#include "lib/utils/Delegate.h"
+
+#include <memory>
+#include <string>
+#include <websocketpp/config/core_client.hpp>
+#include <websocketpp/message_buffer/message.hpp>
+
+namespace bts::websocket {
+
+class WebsocketClientConfig;
+
+class WebsocketClient {
+ public:
+   WebsocketClient(std::string url, std::uint32_t timeout_secs = 20);
+   ~WebsocketClient();
+
+   using MessageHandler = bts::core::Delegate<void(const std::string &)>;
+
+   void connect(const std::map<std::string, std::string> &headers = {});
+   void disconnect();
+   bool is_connected() const;
+   MessageHandler &message_handler();
+   websocketpp::lib::error_code send_text(const std::string &text);
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::websocket
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::websocket {
+
+// clang-format off
+#define FOR_ALL_VALUES(op)                    \
+  op(WebsocketClientState, IDLE)              \
+  op(WebsocketClientState, OPENING)           \
+  op(WebsocketClientState, OPEN)              \
+  op(WebsocketClientState, WAITING_TO_CLOSE)  \
+  op(WebsocketClientState, CLOSING)
+// clang-format on
+
+ENUM_CLASS_DEF(WebsocketClientState, FOR_ALL_VALUES);
+
+#undef FOR_ALL_VALUES
+
+} // namespace bts::websocket
+
+FMT_IMPL(bts::websocket::WebsocketClientState, s, "{}", s.name());
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "lib/core/FMT_IMPL.h"
+#include "lib/utils/ValueWrapper.h"
+
+#include <websocketpp/close.hpp>
+
+namespace bts::websocket {
+
+/**
+ * Simple wrapper to provide a means implement a custom formatter for
+ * `websocketpp::close::status::value` (an alias for a primitive numeric type)
+ */
+using WebsocketCloseStatus = utils::ValueWrapper<
+    struct WebsocketCloseStatusTag,
+    websocketpp::close::status::value>;
+
+} // namespace bts::websocket
+
+FMT_IMPL(
+    bts::websocket::WebsocketCloseStatus,
+    s,
+    "{}",
+    websocketpp::close::status::get_string(*s));
@@ -0,0 +1,7 @@
+#pragma once
+
+#include "lib/core/FMT_IMPL.h"
+
+#include <websocketpp/common/system_error.hpp>
+
+FMT_IMPL(websocketpp::lib::error_code, ec, "{}", ec.message());
@@ -0,0 +1,7 @@
+#pragma once
+
+#include "lib/core/FMT_IMPL.h"
+
+#include <websocketpp/error.hpp>
+
+FMT_IMPL(websocketpp::exception, e, "{}", std::string{e.what()});
@@ -3,6 +3,7 @@
 #include "ShmDumper.h"

 #include "../Shm/MDStructure.h"
+#include "lib/domain_model/Security.h"
 #include "lib/io/JsonDumper.h"
 #include "lib/logging/logging.h"
 #include "lib/utils/SharedList.h"
@@ -51,85 +52,8 @@ string dump_header(const ColdHeader &header) {
   return JsonDumper::dump(header);
 }

-void write_date_string(
-    uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result);
 string ShmDumper::dump_security(const FixedSecurity &security) {
-   ostringstream res;
-   res << "{";
-   res << "'key': " << security.key << ", ";
-   res << "'symbol': '" << security.symbol() << "', ";
-   res << "'type': '" << api::SecurityType_Name(security.type) << "', ";
-   res << "'exchange': '" << api::Venue_Name(security.exchange) << "', ";
-   res << "'alt_symbol': '" << security.alt_symbol() << "', ";
-   res << "'cfi_code': '" << security.cfi_code() << "', ";
-   res << "'security_desc': '" << security.security_desc() << "', ";
-   res << "'platform_id': '" << security.platform_id() << "', ";
-   res << "'display_factor': " << (int)security.display_factor << ", ";
-   res << "'numeric_id': " << security.numeric_id << ", ";
-   res << "'cboe_class_key': " << security.cboe_class_key << ", ";
-   res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
-   res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
-   res << "'price_format': " << (int)security.cme_price_display_format << ", ";
-   res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";
-
-   res << "'price_break': " << security.price_break << ", ";
-   res << "'low_tick': " << security.low_tick << ", ";
-   res << "'high_tick': " << security.high_tick << ", ";
-   res << "'cpp': " << security.cash_per_point << ", ";
-
-   res << "'market_description ': " << security.ice_market_description()
-       << ", ";
-   res << "'cleared_alias ': " << security.ice_cleared_alias() << ", ";
-   res << "'product_description ': " << security.ice_product_name() << ", ";
-   res << "'hub_alias ': " << security.ice_hub_alias() << ", ";
-   res << "'strip_name': " << security.ice_strip_name() << ", ";
-   res << "'mic_code': " << security.mic_code() << ", ";
-
-   switch (security.type) {
-   case SECURITYTYPE_SPOT:
-      break;
-   case SECURITYTYPE_FUTURE: {
-      auto *psec = static_cast<const FixedFutureSecurity *>(&security);
-      res << "'expiration': '";
-      write_date_string(
-          psec->expiration_year, psec->expiration_month, psec->expiration_day,
-          res);
-      res << "' ";
-      break;
-   }
-   case SECURITYTYPE_OPTION: {
-      auto *psec = static_cast<const FixedOptionSecurity *>(&security);
-      res << "'expiration': '";
-      write_date_string(
-          psec->expiration_year, psec->expiration_month, psec->expiration_day,
-          res);
-      res << "', ";
-      res << "'strike': " << psec->strike << ", ";
-      res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' ";
-      break;
-   }
-   case SECURITYTYPE_SPREAD: {
-      auto *psec = static_cast<const FixedSpreadSecurityXL *>(&security);
-      res << "'legs': [";
-      for (size_t x = 0; x < psec->leg_count(); ++x) {
-         auto &leg = psec->legs[x];
-         if (x > 0)
-            res << ",";
-         res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key
-             << ", 'quantity': " << leg.leg_quantity
-             << ", 'leg_price': " << leg.leg_price
-             << ", 'delta': " << leg.leg_delta << ", 'side': '"
-             << api::Side_Name(leg.leg_side) << "' }";
-      }
-      res << "]";
-      break;
-   }
-   default:
-      break;
-   }
-
-   res << "}";
-   return res.str();
+   return JsonDumper::dump(security);
 }

 string dump(datetime::Date date) {
@@ -22,13 +22,14 @@
 #include "lib/domain/TradeRecord.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/common.h"
+#include "lib/io/JsonDumper.h"
 #include "lib/logging/logging.h"
+#include "lib/md/GeneralSecurityIndex.h"
 #include "lib/md/IMarketDataListener.h"
 #include "lib/md/IntPrice.h"
 #include "lib/md/MDProducer.h"
 #include "lib/md/MDProducerHandle.h"
 #include "lib/md/PriceSize.h"
-#include "lib/md/SecurityIDRecorder.h"
 #include "lib/md/SecurityIndex.h"
 #include "lib/md/Shm/FixedSecurity.h"
 #include "lib/md/Shm/MDStructure.h"
@@ -128,8 +129,19 @@ bool update_incremental(const IMarketDataListener::BookOp &op) {
         case IMarketDataListener::BookOpType::Add:
         {
            int insert_position = 0;
-            for (size_t i = 0; i < book_side.size(); i++){//not sure how to access book
-               if(book_side[i].price != 0 && book_side[i].price > op.price ){
+            for (size_t i = 0; i < book_side.size();
+                 i++) { //not sure how to access book
+               if (book_side[i].price == op.price) {
+                  IMarketDataListener::BookOp new_op;
+                  new_op.id = op.id;
+                  new_op.op_type = IMarketDataListener::BookOpType::MODIFY;
+                  new_op.price = op.price;
+                  new_op.quantity = op.quantity;
+                  new_op.side = op.side;
+
+                  return update_incremental(new_op);
+               }
+               if (book_side[i].price != 0 && book_side[i].price > op.price) {
                  //book_side is the book we have ?
                  //op.price is what we received ?
                  insert_position++;
@@ -183,6 +195,16 @@ bool update_incremental(const IMarketDataListener::BookOp &op) {
         case IMarketDataListener::BookOpType::Add:{
            int insert_position = 0;
            for (int i = 0; i < fmin(book_side.size(),5); i++) {
+               if (book_side[i].price == op.price) {
+                  IMarketDataListener::BookOp new_op;
+                  new_op.id = op.id;
+                  new_op.op_type = IMarketDataListener::BookOpType::MODIFY;
+                  new_op.price = op.price;
+                  new_op.quantity = op.quantity;
+                  new_op.side = op.side;
+
+                  return update_incremental(new_op);
+               }
                if (book_side[i].price != 0 && book_side[i].price < op.price) {
                    insert_position++;
                    continue;
@@ -274,10 +296,11 @@ bool update_incremental(const IMarketDataListener::BookOp &op) {
 };

 struct GeneralMarketDataListener::PImpl {
+   const api::domain::Venue venue;
   // if this is set to true, then the security key passed on in the
   // on_securities function is used. If this is not set to true, then a new key
   // is generated
-   const bool use_given_sec_key = true;
+   const bool use_given_sec_key = false;
   const std::string cc_name_prefix = "cc_security";
   size_t n_bk_updates = 0;
   bts::persistence::IDGenerator &id_gen =
@@ -296,12 +319,15 @@ struct GeneralMarketDataListener::PImpl {
   MDArray asks = {};
   IncrementalBookData inc_book_data = {};
   explicit PImpl(
+       api::domain::Venue venue,
       GeneralSecurityIndex &index,
       MDProducer &md_producer,
       std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy)
-       : index(index), md_producer(md_producer), sim_ex_gwy(sim_ex_gwy) {
-      if (sim_ex_gwy != nullptr) {
-         sim_ex_gwy->set_producers(&index);
+       : venue(venue), index(index),
+         md_producer(md_producer),
+         sim_ex_gwy(std::move(sim_ex_gwy)) {
+      if (this->sim_ex_gwy != nullptr) {
+         this->sim_ex_gwy->set_producers(&this->index);
      }
   }
 };
@@ -363,6 +389,7 @@ void copy_books(
 GeneralMarketDataListener::GeneralMarketDataListener(
    GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue)
    : me(std::make_unique<PImpl>(
+          domain::DomainSecurityUtils::convert(venue),
          index,
          md_producer,
          bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway(
@@ -379,7 +406,7 @@ void GeneralMarketDataListener::on_securities(
   LOG(md, Info, "GeneralMarketDataListener::on_securities n_secs: {}",
       complete_security_list.size());
   for (auto s_ptr : complete_security_list) {
-      domain::Security sec = *s_ptr;
+      const domain::Security &sec = *s_ptr;
      const auto &s_comm = sec.security_common;
      CCSecSearchResult search_result = _find_sec(sec, true);
      LOG(md, Info,
@@ -466,21 +493,157 @@ void GeneralMarketDataListener::on_book_increment(
   bts_assert(book_ptr != nullptr);
   _on_book(*book_ptr);
 }
+co2::AggressorSide try_fill_agg_side(
+    const domain::Price &trade_price,
+    const co2::AggressorSide &current_side,
+    const domain::Book *book,
+    const domain::SecurityID &id) {
+   if (current_side != co2::AGGRESSOR_SIDE_NONE)
+      return current_side;
+   if (!book) {
+      LOG(md, Info,
+          "No book found for sec_id: {}. Processing trade with aggressor "
+          "side set to NONE.",
+          id);
+      return current_side;
+   }
+   // Access ask-side and bid-side levels
+   auto book_ask_side = book->asks();
+   auto book_bid_side = book->bids();
+   if (book_ask_side.is_populated() &&
+       trade_price >=
+           book_ask_side[0]
+               .price) //this means the trade happened on the ask side
+   {
+      LOG(md, Info, "Trade Price: {}, Book_Ask[0] Price: {}", trade_price,
+          book_ask_side[0].price);
+      return co2::AGGRESSOR_SIDE_BUY;
+   }
+   if (book_bid_side.is_populated() &&
+       trade_price <=
+           book_bid_side[0]
+               .price) //this means the trade happened on the bid side
+   {
+      LOG(md, Info, "Trade Price: {}, Book_Ask[0] Price: {}", trade_price,
+          book_bid_side[0].price);

+      return co2::AGGRESSOR_SIDE_SELL;
+   }
+   std::string book_str =
+       "Bids: " + bts::io::JsonDumper::dump(book_bid_side, 2) +
+       "\nAsks: " + bts::io::JsonDumper::dump(book_ask_side, 2);
+   LOG(md, Info,
+       "Could not determine Aggressor, Trade Price: {}, Current Book: {}",
+       trade_price, book_str);
+
+   return co2::AGGRESSOR_SIDE_NONE;
+}
 void GeneralMarketDataListener::on_trades(
    const std::vector<domain::TradeRecord const *> &trades) {
   for (auto tp : trades) {
-      auto handle = _find_handle(tp->sec_id);
+
+      std::ostringstream log_stream;
+
+      // Log the important fields from the TradeRecord pointer
+      log_stream << "TradePointer - sec_id: " << tp->sec_id
+                 << ", agg_side: " << tp->agg_side
+                 << ", trade_price: " << tp->trade_price
+                 << ", trade_quantity: " << tp->trade_quantity.value()
+                 << ", time_stamp: " << tp->time_stamp.millis();
+
+      // Log the trade data
+      LOG(md, Diag, "logging tp {}", log_stream.str());
+
+      auto handle =
+          _find_handle(tp->sec_id); // might be the reason behind missing trades
+      auto aggressor_side = convert_agg_side(tp->agg_side);
+
+      // Log the initial aggressor_side
+      LOG(md, Diag, "Initial aggressor side for sec_id: {} is: {}", tp->sec_id,
+          static_cast<int>(aggressor_side));
+
+      if (handle == nullptr) {
+         LOG(md, Info, "No handle found for sec_id: {}", tp->sec_id);
+         continue;
+      }
+      aggressor_side = try_fill_agg_side(
+          tp->trade_price, aggressor_side,
+          me->inc_book_data.get_const_book(tp->sec_id), tp->sec_id);
+
      auto trade_ts = datetime::ll_from_unix_epoch_ms(tp->time_stamp.millis());
+      // Process the trade even if aggressor_side is NONE
      if (handle != nullptr) {
         handle->handle.add_trade(
             utils::TimeStamp::now(),
             PriceSize(
                 convert_price(tp->trade_price), tp->trade_quantity.value()),
-             trade_ts, convert_agg_side(tp->agg_side), 0, false);
+             trade_ts, aggressor_side, 0, false);
+         LOG(md, Diag, "Trade processed for sec_id: {}", tp->sec_id);
      }
   }
 }
+//Needed for diag for trade gaps 
+// void GeneralMarketDataListener::on_trades(
+//     const std::vector<domain::TradeRecord const *> &trades) {
+//    for (auto tp : trades) {
+
+//          for (auto tp : trades) {
+//     std::ostringstream log_stream;
+
+//     // Log the important fields from the TradeRecord pointer
+//     log_stream << "TradePointer - sec_id: " << tp->sec_id
+//                << ", agg_side: " << tp->agg_side
+//                << ", trade_price: " << tp->trade_price
+//                << ", trade_quantity: " << tp->trade_quantity.value()
+//                << ", time_stamp: " << tp->time_stamp.millis();
+
+//     // Log the trade data
+//     LOG(md, Info, "logging tp {}",log_stream.str());}
+
+//       auto handle =
+//           _find_handle(tp->sec_id); //might be the reason behind missing trades
+//       auto aggressor_side = convert_agg_side(tp->agg_side);
+//       if (aggressor_side == co2::AggressorSide::AGGRESSOR_SIDE_NONE) {
+//          auto book = me->inc_book_data.get_const_book(tp->sec_id);
+//          // Check if the book is empty
+//          if (!book) {
+//             LOG(md, Info, "book pointer log");
+//             LOG(md, Info, "Processing trade with sec_id: {}", tp->sec_id);
+//             aggressor_side =
+//                 co2::AggressorSide::AGGRESSOR_SIDE_NONE; // Reset aggressor side
+//             continue; // Skip processing this trade
+//          }else {
+//     LOG(md, Info, "Book found for sec_id: {} with size: {}", tp->sec_id, book->size());
+// }
+
+//          // Access ask-side and bid-side levels
+//          auto book_ask_side = book->asks();
+//          auto book_bid_side = book->bids();
+//           if (!book_ask_side.is_populated() || !book_bid_side.is_populated()) {
+//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_NONE; // Reset
+//               continue; // Skip processing this trade
+//           }
+//            // Determine aggressor side based on trade price
+//           if (tp->trade_price <= book_ask_side[0].price) {
+//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_BUY;
+//           } else if (tp->trade_price >= book_bid_side[0].price) {
+//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_SELL;
+//          }
+//       }
+
+//       //     // Check if ask and bid sides are populated
+
+//       // }
+//       auto trade_ts = datetime::ll_from_unix_epoch_ms(tp->time_stamp.millis());
+//       if (handle != nullptr) {
+//          handle->handle.add_trade(
+//              utils::TimeStamp::now(),
+//              PriceSize(
+//                  convert_price(tp->trade_price), tp->trade_quantity.value()),
+//              trade_ts, aggressor_side, 0, false);
+//       }
+//    }
+// }

 void GeneralMarketDataListener::on_rfqs(
    const std::vector<domain::RFQ const *> &) {}
@@ -503,11 +666,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec(
      if (add_if_absent) {
         uint64_t bts_id = cc_id.value();
         if (!me->use_given_sec_key) {
-            auto logical_key = get_logical_key(sec);
-            auto new_key_opt =
-                me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
-            bts_assert(new_key_opt.has_value());
-            bts_id = new_key_opt.value();
+            bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
         }
         me->cc_sec_id_map.insert({cc_id, {bts_id}});
         LOG(md, Info,
@@ -654,9 +813,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key(
   auto logical_key = get_logical_key(sec);
   // if (me->use_given_sec_key)
   //    return sec.security_common.id.value();
-   auto rv = me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
-   bts_assert(rv.has_value());
-   return rv.value();
+   return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
 }

 void GeneralMarketDataListener::_possibly_match_with_sim_orders(
@@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() {
   return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day());
 }

-void apply_compatibilit_mutations(Security &security) {
-   // Migrate from old tick stuff
-   if (security.has_glbx()) {
-      if (security.has_glbx()) {
-         if (security.contract_spec().main_fraction() == 0) {
-            security.mutable_contract_spec()->set_main_fraction(
-                security.glbx().do_not_use_main_fraction());
-         }
-         if (security.contract_spec().sub_fraction() == 0) {
-            security.mutable_contract_spec()->set_sub_fraction(
-                security.glbx().do_not_use_sub_fraction());
-         }
-         if (security.contract_spec().price_display_format() == 0) {
-            security.mutable_contract_spec()->set_price_display_format(
-                security.glbx().do_not_use_price_display_format());
-         }
-      }
-   }
-}
-
 size_t load_securities(
    std::function<bool(const UserSymbol &symbo)> const &sym_filter,
    const std::function<void(const api::Security &)> &callback) {
@@ -104,7 +84,6 @@ size_t load_securities(
      if (sym_filter(us)) {
         auto blob = select_stmt.getColumn("security_blob");
         if (security.ParseFromArray(blob, blob.getBytes())) {
-            apply_compatibilit_mutations(security);
            bts_assert(security_key == security.id());
            count++;
            callback(security);
@@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) {
         auto result = std::make_shared<Security>();
         auto blob = select.getColumn(0);
         if (result->ParseFromArray(blob, blob.getBytes())) {
-            apply_compatibilit_mutations(*result);
            return result;
         }
      }
@@ -1,18 +1,25 @@
-#include "lib/domain_model/Security.h"
 #ifndef _WIN32
 #include "SecurityIDRecorder.h"
+#include "lib/domain_model/Security.h"
+#include "lib/persistence/IStore.h"
+#include "lib/persistence/SqliteStore.h"
+#include <boost/pool/object_pool.hpp>

 #include "lib/config/config.h"
 #include "lib/logging/logging.h"
 #include "lib/utils/Sqlite_utils.h"
+#include "lib/utils/checked_arithmetic.h"
 #include <boost/filesystem.hpp>
 #include <tbb/concurrent_queue.h>

 #include "lib/config/protos/market_data.pb.h"
 #include "lib/config/protos/system_config.pb.h"

+#include <memory>
 #include <sqlite/sqlite3.h>
+#include <stdexcept>
 #include <string>
+#include <tbb/concurrent_queue.h>

 using namespace std;
 using namespace bts::utils;
@@ -21,22 +28,55 @@ using std::size_t;
 namespace bts {
 namespace md {

+using bts::api::Security;
+using bts::api::SecurityType;
+using bts::api::Venue;
+using bts::config::SystemConfig;
+using bts::persistence::IStore;
+using bts::persistence::SqliteStore;
+
 uint64_t exchange_id(const api::Security &security);

-struct SecurityIDEntry {
-   api::Security security;
-   SecurityIDRecorder::Callback callback;
+static std::unique_ptr<IStore> get_default_store() {
+   return std::make_unique<SqliteStore>("key_ids");
+}
+
+static persistence::StoreRow get_row(
+    Venue venue,
+    SecurityType type,
+    const std::string &name,
+    const std::string &prefix = "") {
+   return persistence::StoreRow{
+       fmt::format(
+           "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6),
+           SecurityType_Name(type).substr(13)),
+       name, ""};
+}
+
+static persistence::StoreRow get_row(
+    const api::Security &s, const std::string &prefix = "") {
+   return get_row(s.exchange(), bts::securities::security_type(s), prefix);
+}
+
+struct SecurityIdRequest {
+   Venue venue;
+   SecurityType type;
+   std::string name;
+   std::function<void(Venue, SecurityType, const std::string &, std::uint32_t)>
+       callback;
 };

 struct SecurityIDRecorderState {
-   SecurityIDRecorderState(SecurityIDRecorderParams &p)
-       : work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close),
-         timer(), lookup_key(p.lookup_key), unresolved_legs() {}
+   SecurityIDRecorderState(
+       std::unique_ptr<IStore> store, KeyLookupOption lookup_key)
+       : store(store == nullptr ? get_default_store() : std::move(store)),
+         pending_handle(), thread_id(), timer(), lookup_key(lookup_key),
+         unresolved_legs() {}

-   tbb::concurrent_queue<std::shared_ptr<SecurityIDEntry>> work;
+   std::unique_ptr<IStore> store{nullptr};
+   tbb::concurrent_queue<std::shared_ptr<SecurityIdRequest>> id_request_queue;
   tbb::concurrent_queue<MDProducerHandle> pending_handle;
   std::thread::id thread_id;
-   sqlite3_ptr db;
   unique_ptr<DispatcherTimer> timer;

   KeyLookupOption lookup_key;
@@ -59,51 +99,41 @@ struct SecurityIDRecorderState {
   void work_handles();
   void work_db();

-   void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
   void assign_key(
       const api::Security &security,
       const SecurityIDRecorder::Callback &key_assigned);
+   void assign_key(
+       api::Venue venue,
+       api::SecurityType type,
+       const std::string &name,
+       std::function<void(
+           api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
+           key_assigned);
+
+   void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
   void add_unresolved_spread(MDProducerHandle &handle);
-   security_key search(const std::string &descriptor);
-   security_key insert(const std::string &descriptor, SecurityType);
+   security_key search(const persistence::StoreRow &row);
+   security_key insert(const persistence::StoreRow &row);
 };

-SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
-    : _self(new SecurityIDRecorderState(params)) {
-   if (params.db) {
-      _self->db = std::move(params.db);
-   }
+SecurityIDRecorder::SecurityIDRecorder(
+    std::unique_ptr<persistence::IStore> store,
+    KeyLookupOption lookup_key,
+    utils::Dispatcher *dispatcher)
+    : _self(std::make_unique<SecurityIDRecorderState>(
+          store == nullptr
+              ? std::make_unique<persistence::SqliteStore>("key_ids")
+              : std::move(store),
+          lookup_key)) {
   init();
-   if (params.dispatcher) {
-      _self->run(*params.dispatcher);
+   if (dispatcher != nullptr) {
+      _self->run(*dispatcher);
   }
 }

 SecurityIDRecorder::~SecurityIDRecorder() {}

-void SecurityIDRecorder::init() {
-   LOG(md, Info, "SecurityIDRecorder::init");
-   string db_dir =
-       config::get<config::SystemConfig>().data_path() + "databases/md/";
-   string db_path = db_dir + "/security_keys.sqlite";
-
-   if (!boost::filesystem::exists(db_dir)) {
-      boost::filesystem::create_directories(db_dir);
-   }
-
-   if (!_self->db) {
-      _self->db = sqlite::open_db(db_path.c_str());
-      bts_assert(_self->db);
-   }
-
-   // Set a long busy timeout
-   sqlite3_busy_timeout(_self->db.get(), 9999);
-   sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
-   sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
-   sqlite::exec_nofail(
-       _self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING "
-                  "NOT NULL PRIMARY KEY, type int);");
-}
+void SecurityIDRecorder::init() {}

 void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
   LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
@@ -113,37 +143,42 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
 }

 void SecurityIDRecorder::assign_key(
-    const api::Security &security,
-    const SecurityIDRecorder::Callback &callback) {
-   LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
-       security.DebugString());
-   if (security.Contract_case() == Security::ContractCase::kSpread) {
-      for (const auto &leg : security.spread().legs()) {
-         if (leg.security().id() == invalid_security_key) {
-            LOG(md, Error, "Security {} has legs which will never be resolved.",
-                security.has_glbx() ? security.glbx().security_desc()
-                                    : security.symbol());
-            // No point in trying to add this to the queue
-            return;
-         }
-      }
-   }
-   _self->assign_key(security, callback);
+    const api::Security &security, const Callback &callback) {
+   assign_key(
+       security.exchange(), bts::securities::security_type(security),
+       bts::securities::descriptor(security),
+       [callback, msec = security](
+           api::Venue, api::SecurityType, const std::string &,
+           uint32_t id) mutable {
+          msec.set_id(id);
+          callback(msec);
+       });
 }

 void SecurityIDRecorderState::assign_key(
-    const api::Security &security,
-    const SecurityIDRecorder::Callback &callback) {
-   LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
-       bts::securities::descriptor(security));
-   if (security.id() == invalid_security_key) {
-      auto work = std::make_shared<SecurityIDEntry>();
-      work->security = security;
-      work->callback = callback;
-      LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}",
-          bts::securities::descriptor(work->security));
-      this->work.push(work);
-   }
+    api::Venue venue,
+    api::SecurityType type,
+    const std::string &name,
+    std::function<
+        void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
+        callback) {
+   LOG(md, Info, "SecurityIDRecorder::assign_key {} {} {}", venue, type, name);
+   auto id_req = std::make_shared<SecurityIdRequest>();
+   id_req->venue = venue;
+   id_req->type = type;
+   id_req->name = name;
+   id_req->callback = callback;
+   id_request_queue.push(id_req);
+}
+
+void SecurityIDRecorder::assign_key(
+    api::Venue venue,
+    api::SecurityType type,
+    const std::string &name,
+    std::function<
+        void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
+        callback) {
+   _self->assign_key(venue, type, name, std::move(callback));
 }

 void SecurityIDRecorder::do_work() { _self->do_work(); }
@@ -220,9 +255,12 @@ void SecurityIDRecorderState::work_handles() {
         retry.push_back(handle);
      } else {
         convert_security(handle.security(), &security);
-         assign_key(security, [handle](const api::Security &security) mutable {
-            handle.set_key(security.id());
-         });
+         assign_key(
+             security.exchange(), bts::securities::security_type(security),
+             bts::securities::descriptor(security),
+             [handle](
+                 api::Venue, api::SecurityType, const std::string &,
+                 uint32_t id) mutable { handle.set_key(id); });
      }
   }

@@ -239,33 +277,30 @@ void SecurityIDRecorderState::work_db() {
   // Then persist that list
   // Once a security has a key it is public so record it in the security
   // database
-   std::shared_ptr<SecurityIDEntry> temp;
-   vector<std::shared_ptr<SecurityIDEntry>> new_entries;
+   std::shared_ptr<SecurityIdRequest> temp;
+   vector<std::pair<uint32_t, std::shared_ptr<SecurityIdRequest>>> new_entries;
   // SPARK-1446 Need to try to add existing securities to the db in case we
   // deleted them by hand
-   vector<std::shared_ptr<SecurityIDEntry>> existing_securities;
-   vector<std::shared_ptr<SecurityIDEntry>> defer;
-   vector<string> new_descriptors;
+   vector<std::shared_ptr<SecurityIdRequest>> existing_securities;
+   vector<persistence::StoreRow> new_rows;
   size_t count = 0;
-   string descriptor;
-   while (work.try_pop(temp)) {
-      descriptor = security_descriptor(temp->security);
-
-      // If we're unable to create a descriptor try again later
-      if (descriptor == "") {
-         defer.push_back(std::move(temp));
-         continue;
+   while (id_request_queue.try_pop(temp)) {
+      security_key key = invalid_security_key;
+      auto row = get_row(temp->venue, temp->type, temp->name);
+      try {
+         key = search(row);
+      } catch (const std::overflow_error &ex) {
+         LOG(persist, Error,
+             "SecurityIDRecorderState::work_db. Row search threw ex: {}",
+             std::string{ex.what()});
      }
-
-      security_key key = search(descriptor);
      if (key != invalid_security_key) {
-         temp->security.set_id(key);
-         temp->callback(temp->security);
+         temp->callback(temp->venue, temp->type, temp->name, key);
         existing_securities.push_back(std::move(temp));
      } else {
         // always push in parallel
-         new_entries.push_back(std::move(temp));
-         new_descriptors.push_back(descriptor);
+         new_entries.push_back(std::make_pair(key, std::move(temp)));
+         new_rows.push_back(std::move(row));
         ++count;
         // Limit the amount of time we spend writing to sqlite during startup
         // to avoid blocking the heartbeat
@@ -275,95 +310,47 @@ void SecurityIDRecorderState::work_db() {
      }
   }

-   for (auto &d : defer) {
-      LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
-          bts::securities::descriptor(d->security));
-      work.push(std::move(d));
-   }
-
   if (!new_entries.empty()) {
-      sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
-
+      store->insert_many_resolve_ids(new_rows);
      for (size_t x = 0; x < new_entries.size(); ++x) {
-         Security &security = new_entries[x]->security;
-         security_key key =
-             insert(new_descriptors[x], securitytype_of(security));
-         bts_assert_d(key);
-         security.set_id(key);
+         auto &entry = new_entries[x];
+         auto &row = new_rows[x];
+         entry.first = row.id;
+         bts_assert_d(entry.first != invalid_security_key);
      }
-
-      sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
   }

   if (!new_entries.empty()) {
      for (size_t x = 0; x < new_entries.size(); ++x) {
-         Security &security = new_entries[x]->security;
-         new_entries[x]->callback(security);
+         auto &[id, req] = new_entries[x];
+         req->callback(req->venue, req->type, req->name, id);
      }
   }
 }

 void SecurityIDRecorder::bulk_insert(
    const std::vector<const api::Security *> &data, const char *prefix) {
-   LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
-   if (data.size() == 0)
-      return;
-   sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
-   std::string descriptor;
-   {
-      auto insert_stmt = sqlite::prepare_query(
-          _self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
-                     "values (?, ?, ?)");
-      for (const api::Security *sec : data) {
-         descriptor = prefix + security_descriptor(*sec);
-         auto type = securitytype_of(*sec);
-
-         sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
-         sqlite3_bind_text(
-             insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
-             SQLITE_STATIC);
-         sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
-         sqlite3_step(insert_stmt.get());
-         sqlite3_reset(insert_stmt.get());
-      }
+   std::vector<persistence::StoreRow> rows;
+   rows.reserve(data.size());
+   for (const auto *s : data) {
+      rows.push_back(get_row(*s, prefix));
   }
-   sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
+   _self->store->insert_many_resolve_ids(rows);
 }

-security_key SecurityIDRecorderState::search(const std::string &descriptor) {
-   LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
-   auto select_stmt = sqlite::prepare_query(
-       db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
-   sqlite3_bind_text(
-       select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
-       SQLITE_STATIC);
-   int result = sqlite3_step(select_stmt.get());
-   if (result == SQLITE_DONE || result == SQLITE_ROW) {
-      return sqlite3_column_int(select_stmt.get(), 0);
+security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) {
+   auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key);
+   if (exists) {
+      if (!val.empty()) {
+         bts_assert(stol(val) == rowid);
+      }
+      return system::math::checked_ltou(rowid);
   }
   return invalid_security_key;
 }

-security_key SecurityIDRecorderState::insert(
-    const std::string &descriptor, SecurityType type) {
-   LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
-   auto insert_stmt = sqlite::prepare_query(
-       db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
-   sqlite3_bind_text(
-       insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
-       SQLITE_STATIC);
-   sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
-   int result = sqlite3_step(insert_stmt.get());
-   if (result == SQLITE_DONE || result == SQLITE_ROW) {
-      return sqlite3_last_insert_rowid(db.get());
-   }
-   // Just in case our insert failed due to duplicate row
-   auto key = search(descriptor);
-   if (key == invalid_security_key) {
-      LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
-          result);
-   }
-   return key;
+security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) {
+   return store->insert(row.ns, row.key, row.value);
 }

 } // namespace md
@@ -2,7 +2,7 @@
 #ifndef _WIN32
 #include "MDProducerHandle.h"
 #include "common.h"
-
+#include "lib/persistence/IStore.h"
 #include "lib/utils/Sqlite_utils.h"
 #include <atomic>
 #include <functional>
@@ -22,26 +22,15 @@ namespace md {

 typedef const std::function<SecurityKey(ExchangeKey)> KeyLookup;
 typedef std::optional<KeyLookup> KeyLookupOption;
-struct SecurityIDRecorderParams {
-   SecurityIDRecorderParams()
-       : dispatcher(), db(sqlite::null_db()), lookup_key() {}
-   SecurityIDRecorderParams(utils::Dispatcher &d)
-       : dispatcher(&d), db(sqlite::null_db()), lookup_key() {}
-   SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key)
-       : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {}
-   SecurityIDRecorderParams(sqlite3_ptr &db)
-       : dispatcher(), db(std::move(db)), lookup_key() {}
-   SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key)
-       : dispatcher(), db(std::move(db)), lookup_key(get_key) {}
-   utils::Dispatcher *dispatcher;
-   sqlite3_ptr db;
-   KeyLookupOption lookup_key;
-};
+
 struct SecurityIDRecorderState;

 class SecurityIDRecorder {
 public:
-   SecurityIDRecorder(SecurityIDRecorderParams);
+   SecurityIDRecorder(
+       std::unique_ptr<persistence::IStore> store = nullptr,
+       KeyLookupOption lookup_key = std::nullopt,
+       utils::Dispatcher *dispatcher = nullptr);
   ~SecurityIDRecorder();
   typedef std::function<void(const api::Security &)> Callback;

@@ -49,6 +38,13 @@ class SecurityIDRecorder {
   void ensure_key(MDProducerHandle handle);
   // Slow (copies security) - for use by services
   void assign_key(const api::Security &security, const Callback &key_assigned);
+   void assign_key(
+       api::Venue venue,
+       api::SecurityType type,
+       const std::string &name,
+       std::function<void(
+           api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
+           key_assigned);

   // If a dispatcher is not provided you must call work periodically to do
   // lookups and flush the reserved ids to disk.
@@ -1,4 +1,4 @@
-#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/Security.h"
 #include "lib/utils/bts_assert.h"
 #ifndef NO_CPP0X

@@ -59,81 +59,6 @@ FixedSpreadLeg::FixedSpreadLeg()
    : key(invalid_security_key), leg_side(Side_MIN), leg_quantity(0),
      leg_price(0), leg_delta(1) {}

-void write_date_string(
-    uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) {
-   result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2)
-          << std::setfill('0') << day << "/" << year - 2000;
-}
-
-void write_leg_descriptor(
-    std::ostringstream &result, char separator, const api::ContractLeg &leg) {
-   result << separator << leg.security().id() << "-q" << leg.quantity() << "p"
-          << leg.price_multiplier() << "d" << leg.leg_delta()
-          << (leg.side() == SIDE_BUY ? "BUY" : "SELL");
-}
-
-std::string security_descriptor(const api::Security &security) {
-   const char separator = ' ';
-   std::ostringstream result;
-   if (security.exchange() == Venue::VENUE_ICE ||
-       security.exchange() == Venue::VENUE_SIMICE)
-      result << security.ice().security_desc() << separator
-             << Venue_Name(security.exchange());
-   else
-      result << security.symbol() << separator
-             << Venue_Name(security.exchange());
-   switch (security.Contract_case()) {
-   case Security::ContractCase::kOption: {
-      result << separator;
-      const auto &o = security.option();
-      write_date_string(
-          o.expiration_year(), o.expiration_month(), o.expiration_day(),
-          result);
-      result << separator << o.strike()
-             << (o.call_put() == PUTORCALL_CALL ? 'C' : 'P');
-      break;
-   }
-   case Security::ContractCase::kFuture: {
-      result << separator;
-      const auto &f = security.future();
-      write_date_string(
-          f.expiration_year(), f.expiration_month(), f.expiration_day(),
-          result);
-      break;
-   }
-   case Security::ContractCase::kSpot: {
-      break;
-   }
-   case Security::ContractCase::kSpread: {
-      // If the security key of a leg is invalid return an error
-      const auto &s = security.spread();
-      bts_assert(s.legs_size() > 0);
-      for (const auto &leg : s.legs()) {
-         if (leg.security().id() == invalid_security_key)
-            return "";
-         write_leg_descriptor(result, separator, leg);
-      }
-      break;
-   }
-   case Security::ContractCase::kSynthetic: {
-      const auto &s = security.synthetic();
-      bts_assert(s.legs_size() > 0);
-      result << separator << s.name()
-             << " pc=" << SyntheticContract::PriceCalc_Name(s.price_calc());
-      for (const auto &leg : s.legs()) {
-         if (leg.security().id() == invalid_security_key)
-            return ""; // unlikey with synthetics
-         write_leg_descriptor(result, separator, leg);
-      }
-      break;
-   }
-   case Security::ContractCase::CONTRACT_NOT_SET:
-      bts_assert(
-          security.Contract_case() != Security::ContractCase::CONTRACT_NOT_SET);
-   }
-   return result.str();
-}
-
 void get_contract_spec(
    const FixedSecurity &source, /*out*/ api::ContractSpecification &spec) {
   spec.set_price_break(source.price_break);
@@ -221,6 +146,10 @@ void convert_security(const FixedSecurity &source, Security *presult) {
      dest.mutable_okx()->set_security_desc(source.security_desc());
      dest.mutable_okx()->set_exchange_id(source.numeric_id);
      break;
+   case Venue::VENUE_TWSE:
+      dest.mutable_twse()->set_security_desc(source.security_desc());
+      dest.mutable_twse()->set_exchange_id(source.numeric_id);
+      break;
   case Venue::VENUE_TAIFEX:
      dest.mutable_taifex()->set_security_desc(source.security_desc());
      dest.mutable_taifex()->set_exchange_id(source.numeric_id);
@@ -292,26 +221,12 @@ void convert_security(const FixedSecurity &source, Security *presult) {
   get_contract_spec(source, *dest.mutable_contract_spec());
 }

-SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
-   switch (cc) {
-   case Security::ContractCase::kFuture:
-      return SECURITYTYPE_FUTURE;
-   case Security::ContractCase::kOption:
-      return SECURITYTYPE_OPTION;
-   case Security::ContractCase::kSpot:
-      return api::SECURITYTYPE_SPOT;
-   case Security::ContractCase::kSpread:
-      return SECURITYTYPE_SPREAD;
-   case Security::ContractCase::kSynthetic:
-      return SECURITYTYPE_SYNTHETIC;
-   case Security::ContractCase::CONTRACT_NOT_SET:
-      bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET);
-   }
-   bts_assert(false);
-}
 SecurityType securitytype_of(const Security &s) {
-   return securitytype_of_contractcase(s.Contract_case());
+   auto type = bts::securities::security_type(s.Contract_case());
+   bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET);
+   return type;
 }
+
 size_t num_legs(const Security &sec) {
   if (sec.Contract_case() == api::Security::ContractCase::kSpread) {
      return sec.spread().legs_size();
@@ -345,6 +260,8 @@ uint64_t exchange_id(const Security &security) {
      return security.binance_usds_futures().exchange_id();
   case Security::ExchangeDataCase::kOkx:
      return security.okx().exchange_id();
+   case Security::ExchangeDataCase::kTwse:
+      return security.twse().exchange_id();
   case Security::ExchangeDataCase::kTaifex:
      return security.taifex().exchange_id();
   // Debatable whether this should be an error or not.
@@ -359,7 +276,7 @@ uint64_t exchange_id(const Security &security) {
 void convert_security(
    const Security &src, FixedSecurity &dest, bool set_security_key) {
   dest.key = set_security_key ? src.id() : invalid_security_key;
-   dest.type = securitytype_of_contractcase(src.Contract_case());
+   dest.type = bts::securities::security_type(src.Contract_case());
   dest.symbol(src.symbol());
   dest.currency(src.currency());
   dest.exchange = src.exchange();
@@ -374,14 +291,19 @@ void convert_security(
      dest.cme_market_segment_id = src.glbx().market_segment_id();
      break;
   case Security::ExchangeDataCase::kCboe:
-      dest.numeric_id = src.deribit().exchange_id();
-      dest.security_desc(src.deribit().security_desc());
+      dest.numeric_id = src.cboe().exchange_id();
+      dest.security_desc(src.cboe().security_desc());
      dest.cboe_class_key = src.cboe().class_key();
      break;
+   case Security::ExchangeDataCase::kTwse:
+      dest.numeric_id = src.twse().exchange_id();
+      dest.security_desc(src.twse().security_desc());
+      // dest.cboe_class_key = src.twse().class_key();
+      break;
   case Security::ExchangeDataCase::kTaifex:
-      dest.numeric_id = src.deribit().exchange_id();
-      dest.security_desc(src.deribit().security_desc());
-      dest.cboe_class_key = src.taifex().class_key();
+      dest.numeric_id = src.taifex().exchange_id();
+      dest.security_desc(src.taifex().security_desc());
+      // dest.cboe_class_key = src.taifex().class_key();
      break;
   case Security::ExchangeDataCase::kCfe:
      dest.cboe_class_key = src.cfe().class_key();
@@ -208,8 +208,6 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS;
 typedef FixedSpreadSecurity<128> FixedSpreadSecurityL;
 typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL;

-std::string security_descriptor(const api::Security &);
-
 // Convert a fixedsecurity to a normal Security - caveat: spreads will be
 // normalized
 Security convert_security(const FixedSecurity &);
@@ -217,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *);
 void convert_security(
    const Security &src, FixedSecurity &dest, bool set_security_key = false);

-SecurityType securitytype_of_contractcase(Security::ContractCase cc);
-SecurityType securitytype_of(const Security &);
-size_t num_legs(const Security &);
-uint64_t exchange_id(const Security &security);
+SecurityType securitytype_of(const api::Security &);
+size_t num_legs(const api::Security &);
+uint64_t exchange_id(const api::Security &security);

 void to_json(nlohmann::json &j, const FixedSpreadLeg &f);

@@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher

 struct SyntheticSecurityService::Pimpl {
   Pimpl(const MetadataStatefulPublisher &fn)
-       : recorder(utils::Dispatcher::background()), synthetics(), on_security(),
-         meta(), on_metadata(), metadata_publisher(fn) {}
+       : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()),
+         synthetics(), on_security(), meta(), on_metadata(),
+         metadata_publisher(fn) {}
   SecurityIDRecorder recorder;
   std::unordered_map<SecurityKey, Security> synthetics;
   utils::Event<Security> on_security;
@@ -0,0 +1,32 @@
+#pragma once
+
+#include "lib/os/spark_data_directory.h"
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/path.hpp>
+
+namespace bts::md {
+
+struct Mdp3DataDirectory {
+   boost::filesystem::path log_dir() const { return root.log_dir() / "onix"; }
+
+   boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; }
+
+   boost::filesystem::path config_file() const {
+      return data_dir() / "config.xml";
+   }
+
+   boost::filesystem::path templates_file() const {
+      return data_dir() / "templates.xml";
+   }
+
+   void create_dirs() {
+      boost::filesystem::create_directories(data_dir());
+      boost::filesystem::create_directories(log_dir());
+   }
+
+ private:
+   spark::SparkDataDirectory root{};
+};
+
+} // namespace bts::md
@@ -142,6 +142,21 @@ message TradeMessage {
   AggressorSide aggressor_side = 9;
   bool from_spread = 10;
 }
+
+enum BboTradeEventType {
+   BBO_TRADE_UNKNOWN = 0;
+   //BOOK_UPDATE = 1;
+   TRADE = 2;
+}
+
+message BboTradeRecord {
+   BboTradeEventType event_type = 1;
+   // Book at time of trade 
+   TotalBook book = 2;
+   // Include trade details only for TRADE events.
+   TradeMessage trade = 3;
+}
+
 message QuoteRequestMessage {
    enum Side {
        UNDEFINED = 0;
@@ -182,6 +197,7 @@ message TotalBook{
 // Message written to the file, for replay
 message RecordedMDMessage{
   uint32 sec_key = 10;
+   string subset = 11;
   int64 timestamp = 20;
   oneof MDRecord {
      bts.api.Security security = 100;
@@ -189,6 +205,7 @@ message RecordedMDMessage{
      TradeMessage trade = 300;      
      QuoteRequestMessage rfq = 400;
      MiscMessage misc = 500;
+      BboTradeRecord bbo_trade = 600;
   }
 }

@@ -194,6 +194,9 @@ struct MDConsumerGroup {
      case api::VENUE_CBOE:
         this->mic = "CBOE";
         break;
+      case api::VENUE_TWSE:
+         this->mic = "TWSE";
+         break;
      case api::VENUE_TAIFEX:
         this->mic = "TAIFEX";
         break;
@@ -9,6 +9,7 @@
 #include "lib/config/config.h"
 #include "lib/metrics/metrics.h"
 #include "lib/os/os.h"
+#include "lib/persistence/SqliteStore.h"
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/Shuffle.h"
 #include "lib/utils/bts_chrono.h"
@@ -160,12 +161,10 @@ class Co2Tests {
 #ifndef _WIN32
   void test_security_id_recorder(const std::string &db_name) {
      // Verify callbacks happen after work complete
-      auto db1 = sqlite::open_db(db_name.c_str());
-      REQUIRE(db1);
-      auto db2 = sqlite::open_db(db_name.c_str());
-      REQUIRE(db2);
-
-      SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
+      SecurityIDRecorder idsvc{
+          std::make_unique<persistence::SqliteStore>(db_name)};
+      SecurityIDRecorder idsvc2{
+          std::make_unique<persistence::SqliteStore>(db_name)};
      security_key result = invalid_security_key;
      Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
      Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
@@ -215,18 +214,18 @@ class Co2Tests {
          std::make_unique<Co2TestInstrument>(s1, cold_header, hot_header);

      // Add spread to SecurityIDRecorder and work it
-      auto db = sqlite::open_db(":memory:");
+      auto db = std::make_unique<persistence::SqliteStore>(":memory:");
      REQUIRE(db);

      SecurityIDRecorder idsvc{
-          {db, [&](ExchangeKey k) -> SecurityKey {
-              for (const auto &fut :
-                   {f1handle->producer(), f2handle->producer()}) {
-                 if (k.value() == fut.security().numeric_id)
-                    return SecurityKey(fut.key());
-              }
-              return SecurityKey(invalid_security_key);
-           }}};
+          std::move(db), [&](ExchangeKey k) -> SecurityKey {
+             for (const auto &fut :
+                  {f1handle->producer(), f2handle->producer()}) {
+                if (k.value() == fut.security().numeric_id)
+                   return SecurityKey(fut.key());
+             }
+             return SecurityKey(invalid_security_key);
+          }};

      auto workwork = [&]() {
         idsvc.do_work();
@@ -4,9 +4,12 @@
 #include "lib/orderman/gateways/CCAPIExecutionGateway.h"
 // #include "lib/orderman/gateways/EMExecutionGateway.h"
 #include "lib/orderman/gateways/FakeExecutionGateway.h"
+#include "lib/orderman/gateways/GeneralExecutionGateway.h"
 #include "lib/orderman/gateways/GLBXExecutionGateway.h"
 #include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
 #include "lib/orderman/gateways/ICEExecutionGateway.h"
+#include "lib/orderman/gateways/IFIXGenericAdapter.h"
+#include "lib/orderman/gateways/OnixFIXAdapter.h"
 #include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
 #include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h"
 #include "lib/tracing/PerfLog.h"
@@ -21,23 +24,30 @@ std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
    const config::JetFIX::Connection &conn,
    trace::PerfLog &pl,
    md::SecurityOracle o) {
+   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
+       utils::pb::to_string(conn));
   if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
      return std::make_unique<SimExecutionGatewayUniqueWrapper>(conn.venue());
   }
-   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
-       utils::pb::to_string(conn));
-   if (conn.venue() == bts::api::VENUE_ICE ||
-       conn.venue() == bts::api::VENUE_SIMICE)
-      return std::make_unique<ICEExecutionGateway>(conn, pl, o);
-   else if (conn.venue() == bts::api::VENUE_GLBX) {
-      if (conn.use_ilink3())
-         return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(
-             conn, pl, o);
-      return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
-   } else if (conn.venue() == bts::api::VENUE_DERIBIT)
-      //   return std::make_unique<EMExecutionGateway>(o);
-      return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
-   return std::make_unique<FakeExecutionGateway>(o);
+   switch(conn.venue()){
+      case bts::api::VENUE_ICE:
+      case bts::api::VENUE_SIMICE:
+         return std::make_unique<ICEExecutionGateway>(conn, pl, o);
+      case bts::api::VENUE_GLBX:{
+         if (conn.use_ilink3())
+            return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>( conn, pl, o);
+         return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
+      }
+      case bts::api::VENUE_DERIBIT:
+         return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
+      case bts::api::VENUE_CBOE:
+         {
+             std::unique_ptr<IFIXGenericAdapter> adapter = std::make_unique<OnixFIXAdapter>(o);
+            return std::make_unique<GeneralExecutionGateway>(conn.venue(),std::move(adapter),o);
+         }
+      default:
+         return std::make_unique<FakeExecutionGateway>(o);
+   }
 }

 } // namespace bts::orderman
@@ -0,0 +1,86 @@
+#include "GeneralExecutionGateway.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/logging/logging.h"
+#include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/OrderMsg.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
+#include <memory>
+#include <type_traits>
+
+//Acts as a gateway, On One side is the ordermanager and on the other onixfix listener and onixfix adapter
+namespace bts::orderman {
+struct GeneralExecutionGateway::PImpl {
+   PImpl(bts::api::Venue venue,
+   std::unique_ptr<IFIXGenericAdapter> adapter,
+   md::SecurityOracle o)
+        : venue(venue), _adapter(std::move(adapter)), sec_oracle(o) {}
+   bts::api::Venue venue;
+   std::unique_ptr<IFIXGenericAdapter> _adapter;
+   md::SecurityOracle sec_oracle;
+   OnExeReport on_exe_report;
+   OnGwStatusChange on_gw_status_change;
+   OnCancelReject on_cancel_reject;
+   OnExchangePositionState on_exchange_position_state;
+   OnExchangeOpenOrdersState on_exchange_open_orders;
+};
+OnGwStatusChange &GeneralExecutionGateway::on_gw_status_change() {
+   return me->on_gw_status_change;
+}
+// Modified constructor to accept the adapter from outside
+GeneralExecutionGateway::GeneralExecutionGateway(bts::api::Venue venue, 
+                                                 std::unique_ptr<IFIXGenericAdapter> adapter, md::SecurityOracle o)
+    : IExecutionGateway(), me(new PImpl(venue, std::move(adapter),o)) {
+    LOG(OrderMan, Info, "ExecutionGateway::ExecutionGateway constructor");
+}
+GeneralExecutionGateway::~GeneralExecutionGateway() {}
+void GeneralExecutionGateway::start() { 
+   me->on_gw_status_change(true);
+   me->_adapter->start(); 
+   me->_adapter->register_callback(me->on_exe_report);
+   LOG(OrderMan,Info,"ExecutionGateway::start started" );
+}
+void GeneralExecutionGateway::send_msgs(
+    std::vector<OrderMsg> const &msgs, trace::CorID) {
+   LOG(OrderMan,Info,"ExecutionGateway::send_msgs" );
+   for (const auto &msg : msgs) {
+      std::visit(
+          [this](auto &&order) {
+             using T = std::decay_t<decltype(order)>;
+             if constexpr (std::is_same_v<T, CreateOrderMsg>) {
+                LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" );
+                me->_adapter->launch_new_order(order);
+             
+             } else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
+                LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" );
+                me->_adapter->cancel_order(order);
+             } else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
+                LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" );
+                me->_adapter->modify_order(order);
+             } else if constexpr (std::is_same_v<T, RFQMsg>) {
+                 // Handle RFQMsg if necessary
+                 // adapter->handle_rfq(order); // Example placeholder
+             }
+          },
+          msg);
+   }
+}
+void GeneralExecutionGateway::dispatch_incoming_fix_messages() {}
+void GeneralExecutionGateway::on_securities(
+    const mercury::Update<NewSecuritiesEvent> &) {}
+void GeneralExecutionGateway::on_admin_data(
+    const mercury::Update<NewAdminDataEvent> &) {}
+void GeneralExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {}
+OnExeReport &GeneralExecutionGateway::on_exe_report() { return me->on_exe_report; }
+OnCancelReject &GeneralExecutionGateway::on_cancel_reject() {
+   static OnCancelReject default_cancel_reject;
+   return default_cancel_reject;
+}
+OnExchangePositionState &GeneralExecutionGateway::on_exchange_position_state() {
+   static OnExchangePositionState default_exchange_position_state;
+   return default_exchange_position_state;
+}
+OnExchangeOpenOrdersState &GeneralExecutionGateway::on_exchange_open_orders() {
+   static OnExchangeOpenOrdersState default_exchange_open_orders;
+   return default_exchange_open_orders;
+}
+} //namespace bts::orderman
@@ -0,0 +1,29 @@
+#pragma once
+#include "lib/orderman/gateways/IExecutionGateway.h"
+#include "lib/orderman/gateways/IFIXGenericAdapter.h"
+namespace bts {
+namespace orderman {
+class GeneralExecutionGateway : public IExecutionGateway {
+public:
+    GeneralExecutionGateway(bts::api::Venue venue,std::unique_ptr<IFIXGenericAdapter> adapter, 
+                                                 md::SecurityOracle o);
+    virtual ~GeneralExecutionGateway();
+    void start() override;
+    void send_msgs(std::vector<OrderMsg> const & msgs, trace::CorID) override;
+    void dispatch_incoming_fix_messages() override;
+    void on_securities(const mercury::Update<NewSecuritiesEvent> &) override;
+    void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
+    void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override;
+    OnExeReport &on_exe_report() override;
+    OnGwStatusChange &on_gw_status_change() override;
+    OnCancelReject &on_cancel_reject() override;
+    OnExchangePositionState &on_exchange_position_state() override;
+    OnExchangeOpenOrdersState &on_exchange_open_orders() override;
+private:
+    struct PImpl;
+    std::unique_ptr<PImpl> me;
+    
+    
+};
+} //namespace bts
+} //namespace orderman
@@ -1,3 +1,6 @@
 #include "IFIXGenericAdapter.h"

-namespace bts::orderman {}
+namespace bts::orderman {
+
+    IFIXGenericAdapter::~IFIXGenericAdapter(){}
+}
@@ -1,22 +1,13 @@
 #pragma once

-#include "lib/api/services/OrderService.pb.h"
-#include "lib/config/protos/jetfix.pb.h"
-#include "lib/domain_model/Security.pb.h"
-#include "lib/jetfix/codec/MessageDefinitions.h"
-#include "lib/jetfix/common/OutFIXPtr.h"
-#include "lib/md/SecurityOracle.h"
-#include "lib/mercury/events/Update.h"
+#include "lib/orderman/CancelOrderMsg.h"
+#include "lib/orderman/CancelReplaceOrderMsg.h"
 #include "lib/orderman/CreateOrderMsg.h"
-#include "lib/orderman/OrderMsg.h"
-#include "lib/orderman/orderman_utils/ExchangeOpenOrders.h"
-#include "lib/orderman/orderman_utils/ExchangePosition.h"
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/utils/bts_no_copy.h"
+#include "lib/tracing/PerfLog.h"

 #include <functional>
-#include <memory>
-#include <variant>
+
+

 namespace bts::orderman {

@@ -29,33 +20,28 @@ class NewAdminDataEvent;
 class ExecutionReport;
 class CancelReject;

-struct OrderStateWithFills {
-   api::orderman::OrderState ordr_state;
-   std::vector<api::orderman::Fill> fills = {};
-};
-
-using OnOrderUpdate = std::function<void(OrderStateWithFills const &)>;
+using OnExecutionReport = std::function<void(ExecutionReport const &, const trace::CorID &)>;

 // fix execution gateway for a single FIX session
 class IFIXGenericAdapter {
 public:
   IFIXGenericAdapter() = default;
-   virtual ~IFIXGenericAdapter() {}
+   virtual ~IFIXGenericAdapter();

   virtual void start() = 0;
-   // this callback should be added in a list, and on receving an update from the other side, it should be passed here
-   virtual void register_callback(const OnOrderUpdate &);
-   virtual void launch_new_order(const CreateOrderMsg &);
-   virtual void modify_order(const CancelReplaceOrderMsg &);
-   virtual void cancel_order(const CancelOrderMsg &);
+//    // this callback should be added in a list, and on receving an update from the other side, it should be passed here
+   virtual void register_callback(const OnExecutionReport &) =0 ;
+   virtual void launch_new_order(const CreateOrderMsg &) =0;
+   virtual void modify_order(const CancelReplaceOrderMsg &) = 0;
+   virtual void cancel_order(const CancelOrderMsg &) =0;
   // as new securities are added
-   virtual void update_securities(
-       const mercury::Update<NewSecuritiesEvent> &) = 0;
-   virtual void update_admin_data(
-       const mercury::Update<NewAdminDataEvent> &) = 0;
+   // virtual void update_securities(
+   //     const mercury::Update<NewSecuritiesEvent> &) = 0;
+   // virtual void update_admin_data(
+   //     const mercury::Update<NewAdminDataEvent> &) = 0;

- private:
-   BTS_NO_COPY(IFIXGenericAdapter);
+//  private:
+//    BTS_NO_COPY(IFIXGenericAdapter);
 };

 } // namespace bts::orderman
@@ -0,0 +1,377 @@
+#include "OnixFIXAdapter.h"
+#include "OnixS/FIXEngine/FIX/Engine.h"
+#include "lib/config/protos/marex_fix_cfg.pb.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/logging/logging.h"
+#include "lib/orderman/CancelOrderMsg.h"
+#include "lib/orderman/CancelReplaceOrderMsg.h"
+#include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/gateways/OnixFIXListener.h"
+#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/helper_macros.h"
+#include <OnixS/FIXEngine/FIX/Exception.h>
+#include <OnixS/FIXEngine/FIX/FIX42.h>
+#include <OnixS/FIXEngine/FIX/Message.h>
+#include <OnixS/FIXEngine/FIX/Numeric.h>
+#include <OnixS/FIXEngine/FIX/ProtocolVersion.h>
+#include <cassert>
+#include <chrono>
+#include <memory>
+#include <quickfix/FixFields.h>
+#include <ratio>
+#include <string>
+#include <unistd.h>
+
+namespace bts::orderman {
+void handleException(
+    const std::string &context, const std::string &error) {
+   bts_assert_msgf(false, "{} Exception: {}", context, error);
+}
+OnixFIXAdapter::OnixFIXAdapter(md::SecurityOracle sec_oracle)
+    : state_handler(std::make_shared<TrafixOrderStateHandler>(sec_oracle)),
+      listener(state_handler) {
+   try {
+      auto onix_cfg = config::get<config::MarexFixCfg>();
+      LOG(OrderMan, Warn, "Initializing OnixFIXAdapter");
+      OnixS::FIX::Engine::
+          init(); // OnixS::FIX::Engine::init() needs to run before initialising session
+      LOG(OrderMan, Diag, "Initializing OnixFIXAdapter engine initailes trying login");
+      session = std::make_unique<Session>(
+          onix_cfg.sender_sub_id(), onix_cfg.target_sub_id(),
+          OnixS::FIX::ProtocolVersion::FIX_42, &listener);
+   } catch (const OnixS::FIX::Exception &ex) {
+      const std::string err = ex.what();
+      LOG(OrderMan, Error, "Initializing OnixFIXAdapter ...{}",err);
+      Engine::shutdown();
+      handleException("OnixFIXAdapter::Construcotr init", err);
+   }
+}
+OnixFIXAdapter::~OnixFIXAdapter() {
+   try {
+      if (session)
+         session->logout();
+      Engine::instance()->unregisterListener();
+      Engine::shutdown();
+   } catch (const OnixS::FIX::Exception &ex) {
+    const std::string err = ex.what();
+      handleException("OnixFIXAdapter::Destructor ", err);
+   }
+}
+void OnixFIXAdapter::start() {
+    auto cfg = config::get<config::MarexFixCfg>();
+    const std::string counterpartyHost = cfg.counterparty_host();
+    const int heartBtInt = cfg.heart_bt_int();
+    const int counterpartyPort = cfg.counterparty_port();
+    const int retryInterval = cfg.retry_interval(); // Add this field in MarexFixCfg
+    const int maxRetries = cfg.max_retries();       // Optional: Set a limit on retries
+
+    int attempt = 0;
+
+    while (true) {
+        try {
+            LOG(OrderMan, Warn, "OnixFIXAdapter::start Attempting login (Attempt: {})", attempt + 1);
+            session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt);
+            LOG(OrderMan, Warn, "OnixFIXAdapter::start Login successful");
+            break; // Exit loop on success
+        } catch (const OnixS::FIX::Exception &ex) {
+            const std::string err = ex.what();
+            LOG(OrderMan, Error, "OnixFIXAdapter::start Login failed: {}. Retrying in {} seconds.", err, retryInterval);
+
+            if (maxRetries > 0 && ++attempt >= maxRetries) {
+                Engine::shutdown();
+                LOG(OrderMan, Error, "OnixFIXAdapter::start Exceeded max retries ({}). Aborting.", maxRetries);
+                handleException("OnixFIXAdapter::start", err);
+                break; // Exit if retries are exhausted
+            }
+            std::this_thread::sleep_for(std::chrono::seconds(std::max(60,retryInterval)));
+        }
+    }
+}
+template <typename T>
+uint translateEnum(T enum_value, const std::map<int, int> &translationMap) {
+   auto it = translationMap.find(enum_value);
+   bts_assert(it != translationMap.end()); // Using assert instead of bts_assert
+   return it->second;
+}
+
+uint translateTimeInForce(api::orderman::TimeInForce time_in_force) {
+   //0 = Day  1 = GTC 2 = At the Open 3 = IOC 4 = FOK 5 = GTX
+   static const std::map<int, int> tifMap = {
+       {api::orderman::TimeInForce::TIME_IN_FORCE_DAY, 0},
+       {api::orderman::TimeInForce::TIME_IN_FORCE_GTC, 1},
+       {api::orderman::TimeInForce::TIME_IN_FORCE_IOC, 3}};
+
+   return translateEnum(time_in_force, tifMap);
+}
+
+uint translateOrderType(api::orderman::NativeOrderType order_type) {
+   //1 = Market 2 = Limit 3 = Stop 4 = Stop Limit 5 = Market on close A = On Close B = Limit on Close
+   static const std::map<int, int> orderTypeMap = {
+       {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_MARKET, 1},
+       {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_LIMIT, 2},
+       {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP, 3},
+       {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP_LIMIT, 4}};
+
+   return translateEnum(order_type, orderTypeMap);
+}
+uint translateSide(api::Side side) {
+   //1 = Buy 2 = Sell 5 = Sell short 6 = Sell short Exempt
+   static const std::map<int, int> sideMap = {
+       {api::Side::SIDE_BUY, 1}, {api::Side::SIDE_SELL, 2}};
+
+   return translateEnum(side, sideMap);
+}
+
+std::string pad_string(
+    const std::string &str, size_t length, char padding = ' ') {
+   if (str.size() > length) {
+      return str.substr(0, length);
+   }
+   return str + std::string(length - str.size(), padding);
+}
+
+// Function to create the OSI Symbol
+static std::string create_osi_symbol(
+    const std::string &symbol,
+    const uint32_t &year,
+    const uint32_t &month,
+    const uint32_t &day,
+    const char &call_put_indicator,
+    const double &strike_price) {
+   if (symbol.size() > 6) {
+      throw std::invalid_argument("Symbol length cannot exceed 6 characters.");
+   }
+
+   std::ostringstream osi_stream;
+
+   // Symbol: 6 bytes, padded with spaces
+   osi_stream << pad_string(symbol, 6);
+
+   // Year: last 2 digits of the year
+   osi_stream << std::setw(2) << std::setfill('0') << (year % 100);
+
+   // Month: 2 digits
+   osi_stream << std::setw(2) << std::setfill('0') << month;
+
+   // Day: 2 digits
+   osi_stream << std::setw(2) << std::setfill('0') << day;
+
+   // Call/Put Indicator: 1 byte
+   osi_stream << call_put_indicator;
+
+   // Strike Price: 5 bytes for dollars, 3 bytes for decimal part
+   int strike_dollars = static_cast<int>(strike_price);
+   int strike_decimals =
+       static_cast<int>((strike_price - strike_dollars) * 1000);
+   osi_stream << std::setw(5) << std::setfill('0') << strike_dollars;
+   osi_stream << std::setw(3) << std::setfill('0') << strike_decimals;
+
+   return osi_stream.str();
+}
+
+void parse_option(Message &order, const api::Security &sec) {
+   auto osi = create_osi_symbol(
+       sec.symbol(), sec.option().expiration_year(),
+       sec.option().expiration_month(), sec.option().expiration_day(),
+       sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? 'c'
+                                                                      : 'p',
+       sec.option().strike());
+   std::ostringstream month_stream;
+   month_stream << std::setw(2) << std::setfill('0')
+                << sec.option().expiration_month();
+   std::string month = month_stream.str();
+   LOG(OrderMan, Warn, "OnixFIXAdapter::parse_option parsing osi:{}", osi);
+   order
+       .set(
+           OnixS::FIX::FIX42::Tags::PutOrCall,
+           sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? "1"
+                                                                          : "0")
+       .set(
+           OnixS::FIX::FIX42::Tags::StrikePrice,
+           std::to_string(sec.option().strike()))
+       .set(
+           OnixS::FIX::FIX42::Tags::MaturityMonthYear,
+           std::to_string(sec.option().expiration_year()) + month)
+       .set(
+           OnixS::FIX::FIX42::Tags::MaturityDay,
+           std::to_string(sec.option().expiration_day()))
+       .set(OnixS::FIX::FIX42::Tags::SecurityType, "OPT")
+       .set(OnixS::FIX::FIX42::Tags::Symbol, sec.symbol());
+}
+
+void add_cfg_fields(Message &order, const config::MarexFixCfg &cfg) {
+   order
+       .set(
+           OnixS::FIX::FIX42::Tags::HandlInst,
+           cfg.handl_inst()) // 1: DMA 2: ALGO 3:CARE
+       .set(
+           OnixS::FIX::FIX42::Tags::Rule80A,
+           cfg.Rule80A_Name(cfg.rule80a())) //A = AGENCY P = PRINCIPAL
+       .set(OnixS::FIX::FIX42::Tags::SenderSubID, cfg.sender_sub_id())
+       .set(OnixS::FIX::FIX42::Tags::TargetSubID, cfg.target_sub_id())
+       .set(OnixS::FIX::FIX42::Tags::ClientID, cfg.client_id())
+       .set(OnixS::FIX::FIX42::Tags::CustomerOrFirm, cfg.customer_or_firm())
+       .set(OnixS::FIX::FIX42::Tags::OpenClose, cfg.open_close());
+   order.set(FIX42::Tags::Currency,cfg.currency());   
+   order.set(FIX42::Tags::SecurityExchange,cfg.security_exchange());
+   order.set(FIX42::Tags::ExDestination,cfg.ex_destination());
+}
+void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) {
+   order.set(OnixS::FIX::FIX42::Tags::Account, msg.routing_record->account())
+       .set(
+           OnixS::FIX::FIX42::Tags::TimeInForce,
+           translateTimeInForce(msg.time_in_force))
+       .set(
+           OnixS::FIX::FIX42::Tags::OrdType, translateOrderType(msg.order_type))
+       .set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side))
+       .set(
+           OnixS::FIX::FIX42::Tags::Price,
+           static_cast<float>(msg.price.value()))
+       .set(OnixS::FIX::FIX42::Tags::OrderQty, msg.quantity.value());
+   if (msg.security->has_option()) {
+      parse_option(order, *msg.security);
+   } else {
+      order.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol())
+          .set(OnixS::FIX::FIX42::Tags::SecurityType, "FUT") //need to fix this
+          .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
+   }
+}
+void OnixFIXAdapter::modify_order(
+    const CancelReplaceOrderMsg &msg) { //confirm will have everything
+   try {
+
+      //use this fileOnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/OnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/doc/html/group__double-decimal-conversion.html
+
+      LOG(OrderMan, Warn, "OnixFIXAdapter::modify_order modifying order{}",
+          msg);
+      auto order_cfg = config::get<config::MarexFixCfg>();
+      auto id =
+          state_handler->get_marex_key(OrderKey(msg.create_fields.key.value()));
+
+      if (id == "") {
+         LOG(OrderMan, Error,
+             "OnixFIXAdapter::modify_order could not find clorid");
+         return;
+      }
+      auto new_id = state_handler->new_id(id);
+
+      Message order(
+          OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Replace_Request,
+          OnixS::FIX::ProtocolVersion::FIX_42);
+      order.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id)
+          .set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order
+          .set(
+              OnixS::FIX::FIX42::Tags::TransactTime,
+              OnixS::FIX::Timestamp::utc(),
+              OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
+
+      add_cfg_fields(order, order_cfg);
+      populate_common_order_fields(order, msg.create_fields);
+      order.validate();
+      LOG(OrderMan, Warn, "order Validated");
+      order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
+      std::string order_string = order.toString('|');
+      state_handler->modifyOrder(msg, new_id);
+      session->send(&order);
+      LOG(OrderMan, Warn, "Order update sent :{}", order_string);
+   } catch (const OnixS::FIX::Exception &ex) {
+     const std::string err = ex.what();
+      handleException("Modify order", err);
+   }
+}
+
+void OnixFIXAdapter::launch_new_order(const CreateOrderMsg &msg) {
+   LOG(OrderMan, Warn, "launch order {}", msg);
+
+   try {
+      LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order creating order");
+      auto order_cfg = config::get<config::MarexFixCfg>();
+
+      auto new_id =
+          state_handler->new_id(":" + std::to_string(msg.key.value()));
+      LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order new id:'{}'",
+          new_id);
+      std::string symbol = msg.security->symbol();
+      Message order(
+          OnixS::FIX::FIX42::Values::MsgType::Order_Single,
+          OnixS::FIX::ProtocolVersion::FIX_42);
+      order
+          .set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order
+          .set(
+              OnixS::FIX::FIX42::Tags::TransactTime,
+              OnixS::FIX::Timestamp::utc(),
+              OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
+      add_cfg_fields(order, order_cfg);
+      populate_common_order_fields(order, msg);
+
+      std::string order_string = order.toString('|');
+      std::string order_json = order.toJson();
+      order.validate();
+      LOG(OrderMan, Warn, "order Validated");
+      order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
+      state_handler->launchOrder(msg, new_id);
+      //   listener.fire_dummy_report(msg);
+      session->send(&order);
+      LOG(OrderMan, Warn, "Order sent :{} json:{}", order_string, order_json);
+
+   }
+   // Shut down all reactors if they exist.
+   catch (const OnixS::FIX::Exception &ex) {
+    const std::string err = ex.what();
+      handleException("Launch order", err);
+   }
+}
+void OnixFIXAdapter::register_callback(const OnExecutionReport &cb) {
+   LOG(OrderMan, Warn, "OnixFIXAdapter::register_callback callback registered");
+   listener.setCallback(cb);
+}
+void OnixFIXAdapter::cancel_order(const CancelOrderMsg &msg) {
+   LOG(OrderMan, Warn, "cancel order {}", msg);
+   try {
+      auto id = state_handler->get_marex_key(OrderKey(msg.key.value()));
+      auto order_cfg = config::get<config::MarexFixCfg>();
+
+      if (id == "") {
+         LOG(OrderMan, Error,
+             "OnixFIXAdapter::modify_order could not find clorid");
+         return;
+      }
+      auto new_id = state_handler->new_id(id);
+      Message order_cancel(
+          OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Request,
+          OnixS::FIX::ProtocolVersion::FIX_42);
+      order_cancel.set(
+          OnixS::FIX::FIX42::Tags::HandlInst, order_cfg.handl_inst());
+      order_cancel.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id);
+      order_cancel.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id)
+          .set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side))
+          .set(
+              OnixS::FIX::FIX42::Tags::TransactTime,
+              OnixS::FIX::Timestamp::utc(),
+              OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
+      if (msg.security->has_option()) {
+         parse_option(order_cancel, *msg.security);
+      } else {
+         order_cancel
+             .set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol())
+             .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
+      }
+      std::string order_str = order_cancel.toString('|');
+      LOG(OrderMan, Warn, "order_cancel Created: {}", order_str);
+      // order_cancel.validate(); fails becasuse of HandlInst
+      LOG(OrderMan, Warn, "order Validated");
+      order_cancel.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
+      state_handler->cancelOrder(msg, new_id);
+      session->send(&order_cancel);
+      LOG(OrderMan, Warn, "order cancel sent");
+
+   }
+   // Shut down all reactors if they exist.
+   catch (const OnixS::FIX::Exception &ex) {
+    const std::string err = ex.what();
+      handleException("Cacnel order", err);
+   }
+}
+} // namespace bts::orderman
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "IFIXGenericAdapter.h"
+#include "lib/orderman/gateways/OnixFIXListener.h"
+#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
+#include <memory>
+
+namespace bts::orderman {
+
+class OnixFIXAdapter : public IFIXGenericAdapter {
+  static constexpr int FIX_TAG_SMART_ROUTING = 847;
+ public:
+   OnixFIXAdapter(md::SecurityOracle sec_oracle);
+   ~OnixFIXAdapter() override;
+   void register_callback(const OnExecutionReport &report) override;
+   void launch_new_order(const CreateOrderMsg &msg) override;
+   void cancel_order(const CancelOrderMsg &msg) override;
+   void modify_order(const CancelReplaceOrderMsg &) override;
+   void start() override;
+
+ private:
+   std::shared_ptr<TrafixOrderStateHandler> state_handler;
+   OnixFixListener listener;
+   std::unique_ptr<Session> session;
+};
+
+} // namespace bts::orderman
@@ -0,0 +1,264 @@
+#include "OnixFIXListener.h"
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
+#include "lib/logging/logging.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
+#include "lib/orderman/gateways/IFIXGenericAdapter.h"
+#include "lib/orderman/orderman_utils/ExecutionReport.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/utils/Uuid.h"
+#include "lib/utils/bts_assert.h"
+#include <OnixS/FIXEngine/FIX/FIX42.h>
+#include <OnixS/FIXEngine/FIX/Tag.h>
+#include <string>
+
+namespace bts::orderman {
+ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) {
+    ExecutionReport *er_p = exe_report_pool.construct();
+    ExecutionReport &er = *er_p;
+    std::string account_name = "BTSIM";
+    std::array<char, 12> an_array = {};
+    size_t i = 0;
+    for (const char c : account_name) {
+    an_array[i++] = c;
+    }
+    er.account_name = AccountName(an_array);
+    to_array(msg.routing_record->account(), &er.account_name.value());
+    to_array(
+        msg.routing_record->operator_id(), &er.operator_id.value());
+    er.order_key = msg.key;
+    er.msg_seq_num = 0;
+    er.security = msg.security;
+    er.leg_security = nullptr;
+    er.side = msg.side;
+    er.time_in_force = msg.time_in_force;
+    er.order_type = msg.order_type;
+    er.exec_type = api::orderman::EXEC_TYPE_NEW;
+    er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
+    er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
+    auto s = std::to_string(msg.key.value());
+    std::array<char, 17> idAsArray{};
+    std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin());
+    er.exchange_order_id = ExchangeOrderID{idAsArray};
+    er.price = msg.price;
+    er.quantity = msg.quantity;
+    er.filled_quantity = Quantity(0);
+    er.last_shares = Quantity(0);
+    er.msg = "Message Placed";
+    er.exchange = msg.security->exchange();
+    // Set transaction time
+    ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
+    er.transact_time = now;
+    cb(er,{});
+    return er;
+}
+
+namespace {
+    ExecutionReport createDefaultExecutionReport() {
+        ExecutionReport er;
+        er.exchange = api::VENUE_CBOE;  
+        std::string account_name = "BTSIM";
+        std::string ops_id = "op_id_bts";
+
+        std::array<char, 12> account_array = {};
+        std::copy_n(account_name.begin(), std::min(account_name.size(), account_array.size()), account_array.begin());
+        er.account_name = AccountName(account_array);
+
+        to_array(ops_id, &er.operator_id.value());
+        return er;
+    }
+
+    void populateExecTypeAndOrderStatus(ExecutionReport &er, int exec_type_value) {
+        switch (exec_type_value) {
+            case 0:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_NEW;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
+                break;
+            case 1:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_PARTIAL_FILL;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_PARTIALLY_FILLED;
+                break;
+            case 2:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_FILL;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_FILLED;
+                break;
+            case 4:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_CANCELED;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
+                break;
+            case 5:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED;
+                break;
+            case 8:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+                break;
+            default:
+                er.exec_type = api::orderman::ExecType::EXEC_TYPE_UNDEFINED;
+                er.order_status = api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
+                break;
+        }
+    }
+
+    void populateCreateOrderFields(ExecutionReport &er, const CreateOrderMsg &create_order_msg) {
+        er.security = create_order_msg.security;
+        er.order_type = create_order_msg.order_type;
+        er.time_in_force = create_order_msg.time_in_force;
+        er.manual_order_indicator = create_order_msg.manual_order_indicator;
+
+        to_array(create_order_msg.routing_record->account(), &er.account_name.value());
+        to_array(create_order_msg.routing_record->operator_id(), &er.operator_id.value());
+        er.exchange = create_order_msg.security->exchange();
+    }
+
+    std::array<char, 17> stringToFixedArray(const std::string &s) {
+        std::array<char, 17> idAsArray{};
+        std::copy_n(s.begin(), std::min(s.size(), idAsArray.size() - 1), idAsArray.begin());
+        return idAsArray;
+    }
+}
+ExecutionReport OnixFixListener::get_exe_report(Message msg) {
+    ExecutionReport *er_p = exe_report_pool.construct();
+    ExecutionReport &er = *er_p;
+
+    auto msg_str = msg.toString();
+    LOG(OrderMan,Info, "OnixFixListener::get_exe_report {}", msg_str);
+
+    // Extract order key and create order message
+    er.order_key = state_handler->getOrderKey(msg.get(FIX42::Tags::ClOrdID).toString()); //OrderKey(extractUInt(msg.get(FIX42::Tags::ClOrdID).toString()));
+    auto m = state_handler->getCreateOrderMsg(er.order_key);
+
+    if(m.has_value()){
+        populateCreateOrderFields(er, m.value());
+    }else{ //handle this later
+        LOG(OrderMan,Error, "OnixFixListener::get_exe_report {} could not find create order message", msg_str);
+        er = createDefaultExecutionReport();
+    }  
+    // Populate fields from the message
+
+    int tmp;
+    msg.get(FIX42::Tags::MsgSeqNum).toNumber(tmp);
+    er.msg_seq_num = tmp;
+
+    er.leg_security = nullptr;
+
+    er.side = msg.get(FIX42::Tags::Side).toString()=="1"?api::Side::SIDE_BUY:api::Side::SIDE_SELL;
+
+    int exec_type_value = 9; // Default to undefined
+    msg.get(FIX42::Tags::ExecType).toNumber(exec_type_value);
+    populateExecTypeAndOrderStatus(er, exec_type_value);
+    
+    auto price_str = msg.get(FIX42::Tags::Price);
+    er.price = Price(price_str.valid() ? std::stod(price_str.toString()) : 0);
+
+    auto order_qty_str = msg.get(FIX42::Tags::OrderQty);
+    er.quantity = Quantity(order_qty_str.valid() ? std::stoi(order_qty_str.toString()) : 0);
+
+    auto leaves_qty_str = msg.get(FIX42::Tags::LeavesQty);
+    er.filled_quantity = leaves_qty_str.valid() ? er.quantity - Quantity(std::stoi(leaves_qty_str.toString())) : er.quantity;
+
+    auto last_shares_str = msg.get(FIX42::Tags::LastShares);
+    er.last_shares = Quantity(last_shares_str.valid() ? std::stoi(last_shares_str.toString()) : 0);
+
+    auto avg_px_str = msg.get(FIX42::Tags::AvgPx);
+    er.avg_price = Price(avg_px_str.valid() ? std::stod(avg_px_str.toString()) : 0);
+
+    auto text_str = msg.get(FIX42::Tags::Text).toString();
+    er.msg = text_str.substr(0, std::min(text_str.length(), size_t(15)));
+
+    auto exec_id_str = msg.get(FIX42::Tags::ExecID);
+    std::string fill_id = exec_id_str.valid() ? exec_id_str.toString() : bts::utils::Uuid().to_text_string();
+    to_array(fill_id, &er.exchange_fill_id.value());
+
+    auto last_px_str = msg.get(FIX42::Tags::LastPx);
+    er.last_price = Price(last_px_str.valid() ? std::stod(last_px_str.toString()) : 0);
+
+
+    er.exchange_order_id = ExchangeOrderID{stringToFixedArray(msg.get(FIX42::Tags::OrderID).toString())};
+    
+    
+    // er.transact_time = ExchangeTimeStamp64(datetime::unix_nano_now());
+    //exchgane gives this "TransactTime": "20241111-16:36:08.243",
+    // er.transact_time = 
+    // Set transaction time
+    ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now()); 
+    er.transact_time = now;
+
+    return er;
+}
+
+void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) {
+    std::string msg_str = msg.toString();
+    std::string msg_json = msg.toJson();
+
+        FieldValueRef t = msg.type();
+        char message_type;
+        if(!t.toChar(message_type)){
+            LOG(OrderMan, Warn, 
+        "no message type message: {} As json {}",
+        msg_str, msg_json);
+            return;
+        }
+    LOG(OrderMan, Warn, 
+        "Incoming application-level message: {} As json {}, type: {}",
+        msg_str, msg_json,message_type);
+
+        switch (message_type) {
+            case '8':
+                {
+                    //exection report
+                    auto er = get_exe_report(msg);
+                    if(er.security==nullptr){
+                    LOG(OrderMan, Error,"DID not have sec"); 
+                    return;
+                    }
+                    LOG(OrderMan, Error,"msg {}",er.msg); 
+                    cb(er,{});
+                }
+        }
+
+    // LOG(OrderMan, Warn,
+    //     "onInboundApplicationMsg::inbound firing er: {}",
+    //     er->to_string());
+
+    // cb(*er, {});
+}
+
+
+void OnixFixListener::onInboundSessionMsg(Message & msg, Session * /*sn*/)
+{
+    std::string msg_str = msg.toString();
+    std::string msg_json = msg.toJson();
+    LOG( OrderMan, Warn,  
+    "Incoming session-level message:\n {} As json {}",
+    msg_str,
+    msg_json);
+}
+
+void OnixFixListener::onStateChange(SessionState::Enum newState, SessionState::Enum prevState,
+                             Session * /*sn*/)
+{
+    std::string prevst = SessionState::toString(prevState);
+    std::string newst = SessionState::toString(newState);
+    LOG(OrderMan, Warn,  
+    "Session's state is changed, prevState = {} , newState= {}", 
+    prevst,
+    newst);
+}
+
+void OnixFixListener::onError(ErrorReason::Enum /*reason*/, const std::string & description, Session * /*sn*/)
+{
+    LOG( OrderMan, Error, "Session-level error: {}", description);
+}
+
+void OnixFixListener::onWarning(WarningReason::Enum /*reason*/, const std::string & description,
+                         Session * /*sn*/)
+{
+    LOG( OrderMan, Error, "Session-level warning: {}", description);
+}
+
+
+}   // namespace bts::orderman
@@ -0,0 +1,52 @@
+#pragma once
+
+#include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
+#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
+#include "lib/orderman/orderman_utils/ExecutionReport.h"
+#include <OnixS/FIXEngine.h>
+#include <OnixS/FIXEngine/FIX/Message.h>
+#include <boost/pool/object_pool.hpp>
+
+namespace bts::orderman {
+class TrafixOrderStateHandler;
+using namespace OnixS::FIX;
+class OnixFixListener : public OnixS::FIX::ISessionListener {
+ public:
+   OnixFixListener( std::shared_ptr<TrafixOrderStateHandler> state_handler) : state_handler(state_handler),cb(dummy) {
+      LOG(OrderMan, Info, "OnixFixListener::cb set callback registered");
+   }
+   std::shared_ptr<TrafixOrderStateHandler> state_handler;
+   unsigned long exchange_oid = 0;
+   /// Is called when the application-level message is received from the counterparty.
+   void onInboundApplicationMsg(Message &msg, Session *sn)
+       ONIXS_FIXENGINE_FINAL;
+   // Set the real callback function
+   void setCallback(const OnExeReport &callback) {
+      LOG(OrderMan, Info, "OnixFixListener::setCallback callback registered");
+      cb = callback;
+   }
+   /// Is called when the session-level message is received from the counterparty.
+   void onInboundSessionMsg(Message &msg, Session *sn) ONIXS_FIXENGINE_FINAL;
+   void onStateChange(
+       SessionState::Enum newState,
+       SessionState::Enum prevState,
+       Session *sn) ONIXS_FIXENGINE_FINAL;
+   OnExeReport cb;
+   ExecutionReport fire_dummy_report(CreateOrderMsg msg);
+   ExecutionReport get_exe_report(Message msg);
+   static void dummy(const ExecutionReport &, const trace::CorID &) {
+      LOG(OrderMan, Error, "OnixFixListener::Dummy fired");
+   }
+   void onError(
+       ErrorReason::Enum reason,
+       const std::string &description,
+       Session *sn) ONIXS_FIXENGINE_FINAL;
+   void onWarning(
+       WarningReason::Enum reason,
+       const std::string &description,
+       Session *sn) ONIXS_FIXENGINE_FINAL;
+   boost::object_pool<ExecutionReport> exe_report_pool;
+};
+
+} // namespace bts::orderman
@@ -0,0 +1,88 @@
+#include "TrafixOrderStateHandler.h"
+#include "lib/logging/logging.h"
+#include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/order_factory/IOrderFactory.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <string>
+
+namespace bts::orderman {
+
+TrafixOrderStateHandler::TrafixOrderStateHandler(md::SecurityOracle sec_oracle)
+    : pimpl_(std::make_unique<PImpl>(sec_oracle)) {}
+
+TrafixOrderStateHandler::~TrafixOrderStateHandler() = default;
+
+TrafixOrderStateHandler::OrderInfo* TrafixOrderStateHandler::get_order_info(const OrderKey&key) const {
+    auto it = pimpl_->orders.find(key);
+    if (it != pimpl_->orders.end()) {
+        return &it->second;
+    }
+    return nullptr;
+}
+
+
+OrderKey TrafixOrderStateHandler::launchOrder(const CreateOrderMsg& msg, const std::string& clor_id) {
+    std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
+    pimpl_->orders.emplace(msg.key, OrderInfo(msg, clor_id));
+    return msg.key;
+}
+OrderKey TrafixOrderStateHandler::modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id) {
+    std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
+    LOG(md,Info,"Modifying Order: {}, id:{}",msg,clor_id);
+    auto v = get_order_info(msg.create_fields.key);
+    if(v==nullptr) return null_order_key();
+    LOG(md,Info,"Modifying found order info Order: {}, id:{}",msg,clor_id);
+    v->update(msg, clor_id);
+    return msg.create_fields.key;
+}
+OrderKey TrafixOrderStateHandler::cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id) {
+    std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
+    LOG(md,Info,"canceling Order: {}, id:{}",msg,clor_id);
+    auto v = get_order_info(msg.key);
+    if(v==nullptr) return null_order_key();
+    std::string cancel_id = clor_id;
+    v->cancel(msg, cancel_id);
+    return msg.key;
+}
+//static
+std::string TrafixOrderStateHandler::new_id(const std::string& current_id) {
+   LOG(OrderMan, Warn, "new_id {}", current_id);
+    int update_number = getUpdateNumber(current_id)+1;
+   LOG(OrderMan, Warn, "new_id {}, update number {}", current_id,update_number);
+
+    return std::to_string(update_number) + ":" + std::to_string(getOrderKey(current_id).value());
+}
+OrderKey TrafixOrderStateHandler::getOrderKey(const std::string& id) {
+    std::size_t pos = id.find(':');
+    if (pos == std::string::npos) return null_order_key();
+    return OrderKey(std::stoi(id.substr(pos + 1)));
+}
+int TrafixOrderStateHandler::getUpdateNumber(const std::string& id) {
+    std::size_t pos = id.find(':');
+    if(pos==std::string::npos || pos ==0){
+        return 0;
+    }
+    std::string prefix = id.substr(0, pos);
+    bts_assert_msgf(std::all_of(prefix.begin(), prefix.end(), ::isdigit), "Failed to get update number id:{}",id);
+    return  std::stoi(prefix);
+}
+
+std::optional<CreateOrderMsg> TrafixOrderStateHandler::getCreateOrderMsg(const OrderKey& key) const {
+     std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
+    auto v = get_order_info(key);
+    if(v!=nullptr) return v->order_msg;
+    LOG(md, Error, "Order with key {} not found.", key);
+    return std::nullopt;
+}
+std::string TrafixOrderStateHandler::get_marex_key(const OrderKey &key) {
+     std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
+    
+    auto v = get_order_info(key);
+    if(v!=nullptr) return v->current_id;
+    return "";
+}
+
+} // namespace b
Author	SHA1	Message	Date
Adeel Khan	6c1fe0f561	Fix: Taifex Time Conversion	2025-03-26 18:49:17 +00:00
Adeel Khan	a0eaf60eec	FIX: Taifex Crash Fixed, Added Diag Logs	2025-03-26 18:48:27 +00:00
Adeel Khan	f2080d558a	ADD: TBBO & Symbol wise Naming in Recorder	2025-03-26 18:25:28 +00:00
Adeel Khan	06bfe9c29f	ADD: TWSE Code Complete	2025-03-26 18:20:39 +00:00
Adeel Khan	022546a6c1	Revert "Diag: Pushing changes prior to 6th March Crash" This reverts commit `344d180557`.	2025-03-26 17:59:59 +00:00
Adeel Khan	344d180557	Diag: Pushing changes prior to 6th March Crash	2025-03-06 05:56:01 +00:00
Adeel Khan	a485f8139c	ADD: TWSE to WebApp	2024-12-26 21:53:35 -05:00
Adeel Khan	f50066f4d2	ADD: TWSE throughout Sparkd	2024-12-26 21:53:35 -05:00
Taha	277c3056c4	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-12-09 15:37:50 -05:00
Taha	5eb95e69cb	Cleaning up some streams,new prompt etc	2024-12-09 15:37:45 -05:00
John Colvin	058e11eadc	fix build: Fix install path for sparkd	2024-12-05 18:31:36 +00:00
John Colvin	6ff52e7107	fix build: Add onixs libs again Taha accidentally reverted externals	2024-12-04 16:56:46 +00:00
John Colvin	ee4c28ed41	Port ws client over from cme-websocket-connector Need this to implement execution at crypto exchanges (beginning with Binance and OKX)	2024-12-04 16:56:46 +00:00
Mohammad Arham	50d26f0060	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-12-03 09:52:09 -05:00
Mohammad Arham	36378e6074	Add:: Recording md to sql	2024-12-03 14:51:32 +00:00
John Colvin	8cde8ee6e1	build: Add script to publish the api server This will build the app as a self-contained executable here: build/Release/bin/net7.0/linux-x64/publish	2024-11-28 16:35:21 +00:00
John Colvin	4f8fad444e	fix build: Add missing csproj deps to Spark.Api.Server.sln	2024-11-28 16:35:20 +00:00
John Colvin	832c645a30	build: Map cmake install dir to associated preset Cpp bins are now installed to "install/<cmake_preset>/bin". For example, "install/sparkd-release-gcc/bin", "install/spark-api-server-linux/bin", etc.	2024-11-28 16:35:20 +00:00
John Colvin	691343d9b6	feat: Add datastreams to spark api server	2024-11-28 06:14:00 +00:00
Taha	461cbad338	Adding FakeBBO stream	2024-11-27 13:36:14 -05:00
Taha	fecbcbb6b6	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-11-26 22:33:31 -05:00
Taha	aab8e83543	Some rewrite of streams that fetch bars, and making the other business streams thin wrapper over stream primitives	2024-11-26 22:33:26 -05:00
Mohammad Arham	420b46741e	ADD: Config for sql writer in mdrecorder	2024-11-24 19:14:45 +00:00
Mohammad Arham	e8f65f95bd	Wrapped proto write in function in mdrecorder	2024-11-24 19:13:36 +00:00
Mohammad Arham	0e893b9b5d	Fix: Multiple Exchange data sources work without timee	2024-11-20 23:48:50 +00:00
Mohammad Arham	0593ae0320	Fix: Logging path was incorrect	2024-11-20 23:48:24 +00:00
Mohammad Arham	960149b9c5	Fix: Stupid bug on sleep time	2024-11-20 22:49:22 +00:00
Mohammad Arham	b5f5f2f3d5	Merge branch 'development' of bitbucket.org:bluetradesys/spark into development	2024-11-20 17:21:45 -05:00
John Colvin	bd8c97244a	refactor: Return row id in IStore::retrieve_one This is a prerequisite to synchronizing security ids stored via IDGenerator (the "new" way to store ids, added when we implemented Deribit), and SecurityDB (the original way from spark days). The problem was that these 2 implementations were using separate databases. We need to unify the interfaces between these storage services to avoid breaking existing code.	2024-11-20 22:17:08 +00:00
John Colvin	cb187e6997	Interpret db strng param as name or path in SqliteStore ctr	2024-11-20 22:17:08 +00:00
John Colvin	4004ed2267	refactor: Normalize sec key indexing for old md sources Now both old and new (domain) md sources index securities by the same string format.	2024-11-20 22:17:08 +00:00
John Colvin	a00347ac3f	Add ostream operator for Venue	2024-11-20 22:17:08 +00:00
John Colvin	c4d08840bd	Add ostream operator for SecurityType	2024-11-20 22:17:08 +00:00
John Colvin	84c0684b63	refactor: Normalize sec key indexing for new (domain) md sources This is part of a larger effort to unify the way we index securities in order to generate unique ids. Older md sources (e.g., MDP3Listener) use SecurityIDRecorder to generate ids, which uses a different database and different key-string formatting than the IDGenerator used by the newer md sources (e.g., GeneralMarketDataListener and various ccapi-based sources).	2024-11-20 22:17:08 +00:00
Mohammad Arham	bca8df6d99	Fix: Release Crash fixed still has timer	2024-11-20 22:14:48 +00:00
Mohammad Arham	98f8e58f3f	Add optional tags to config	2024-11-20 22:12:06 +00:00
Mohammad Arham	76f18f3443	Fix: typo in fixed securities	2024-11-20 20:17:29 +00:00
Mohammad Arham	b67d748ba1	removed: moved onix glbx to main controler thread	2024-11-20 06:58:14 +00:00
Mohammad Arham	0113721355	Add: Config for marex reconnect attemps	2024-11-20 06:29:41 +00:00
Mohammad Arham	715f7f7c70	Fix: Race condition causes crash not sure if fixed	2024-11-20 06:17:28 +00:00
Mohammad Arham	89955597c1	Add: Onix/Marex fix adapter, listener, and state handler	2024-11-20 06:16:58 +00:00
Mohammad Arham	96d9aae048	Add: IFIXGenericAdapter	2024-11-20 04:45:36 +00:00
Mohammad Arham	f28d32946f	Add: General Execution Gateway (between system and IFIXAdapter )	2024-11-20 04:39:43 +00:00
Mohammad Arham	924efb2c3f	Add: Added proto for marex fix config	2024-11-20 04:26:39 +00:00
Mohammad Arham	50abb1d784	Fix: Refactored to Switch Statement instead of if and aadded cboe	2024-11-20 04:24:51 +00:00
Mohammad Arham	b5ef39ee4b	Fix: Change default tick_config for taifed	2024-11-20 04:18:58 +00:00
Mohammad Arham	5b61b4fe69	Add: Accounts for CBOE Execution, subaccounts dont work	2024-11-20 03:22:18 +00:00
Mohammad Arham	2d8523ae67	FIX: Order cancel now working in emnative order	2024-11-20 03:18:31 +00:00
Mohammad Arham	f0eef447d3	Add: Exchange parameter added to EMNAtive Order	2024-11-20 03:18:04 +00:00
Mohammad Arham	8604d5672c	ADD: Comments for CME execution for later	2024-11-20 03:11:54 +00:00
Mohammad Arham	30822fb1f1	Fix: move start glbx onix to seperate thread	2024-11-20 03:09:07 +00:00
Mohammad Arham	826bbf76a9	Add: load_reference_data config for databento	2024-11-20 02:53:19 +00:00
John Colvin	eb8c97a1b7	fix: Detect integer ticks of all sizes (not just 1) Also don't divide by zero! Bugs (fixed): * Spark crashes when opening a ladder with integer tick size of 5. This was first observed with TAIFEX futures. Cause: The ladder throws a DivideByZero exception when calculating the number of ticks per dollar. It isn't caught until it bubbles up to the scope in which 'Dispatcher.Run' was called. The catch handler then deliberately assert crashes.	2024-11-12 16:15:47 -05:00
John Colvin	5517c58724	fix (ui): Display trade time w/ microsecond precision Was only showing milliseconds before	2024-11-12 11:42:23 -05:00
John Colvin	e9bcdd0c65	fix build: Update Newtonsoft.Json to 13.0.3 in Spark.csproj	2024-11-12 10:35:51 -05:00