Merge branch 'development.activ_md' of bitbucket.org:bluetradesys/spark into development.activ_md

WARNING: This breaks the sparkd clang build (only works with gcc)

CMakePresets.json

@@ -113,8 +113,8 @@
       "description": "Debug build compiled w/ gcc",
       "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc",
-        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_C_COMPILER": "gcc-12",
+        "CMAKE_CXX_COMPILER": "g++-12",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,6 +15,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,13 +11,27 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,6 +37,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,17 +1,13 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class MinStream : ViewStream
-//    {
-//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
-//       {
-//          mySetRealSource (new GroupByStream (stream, 
-//                            new MinGroupByEvaluator (outputCol, inputCol)));
-//       }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
 
-//       public MinStream (ViewStream stream, string outputCol)
-//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-//    }
-// }
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,94 +1,82 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Computes the sum for a given column
-//    /// </summary>
-//    public class SumGroupByEvaluator : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the sum for a given column
+    /// </summary>
+    public class SumGroupByEvaluator : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-         
-//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
+            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
 
-//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
-//       {
-//          myOutputCols = output_columns;
-//          myGroupByCols = groupby_columns;
-//       }
+        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
+            myOutputCols = output_columns;
+            myGroupByCols = groupby_columns;
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-//    public class SumGroupByEvaluator<T> : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-//          : base(outputCol, groupByCols)
-//       { }
+    public class SumGroupByEvaluator<T> : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-//          : base(colMapEpression, outputCol, groupByCols) { }
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate(DataPacket dp)
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
-//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-// }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,107 +1,89 @@
-// using System;
-// using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
-// namespace BCG
-// {
-//    public class NullMapper : IStreamMapper
-//    {
-//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+namespace BCG {
+    public class NullMapper : IStreamMapper {
+        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-//       public NullMapper(string[] valid_cols)
-//       {
-//          foreach (string c in valid_cols)
-//             _valid_cols[c] = true;
-//          _valid_cols["id"] = true;
-//       }
+        public NullMapper(string[] valid_cols) {
+            foreach (string c in valid_cols)
+                _valid_cols[c] = true;
+            _valid_cols["id"] = true;
+        }
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-//       public IBlueRow map(IBlueRow row)
-//       {
-//          MutableBlueRow out_row = new MutableBlueRow(row);
-//          foreach (string c in row.schema.columnNames)
-//             if (!_valid_cols.ContainsKey(c))
-//                out_row.setValue(c, DBNull.Value);
-//          return out_row;
-//       }
-//    }
+        public IBlueRow map(IBlueRow row) {
+            MutableBlueRow out_row = new MutableBlueRow(row);
+            foreach (string c in row.schema.columnNames)
+                if (!_valid_cols.ContainsKey(c))
+                    out_row.setValue(c, DBNull.Value);
+            return out_row;
+        }
+    }
 
-//    public class SumStream : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//       {
-//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//       }
+    public class SumStream : ViewStream {
+        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns)
-//       {
-//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-//          rs = new ProjectStream(rs, "!fake_col");
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns) {
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
+            mySetRealSource(rs);
+        }
 
-//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
-//       {
-//          string[] output_columns = space_seperated_output_columns.Split(' ');
-//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
-//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
-//       }
+        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
+            string[] output_columns = space_seperated_output_columns.Split(' ');
+            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
-//       {
-//          ViewStream rs = null;
-//          groupby_columns = groupby_columns ?? new string[] {};
-//          if (groupby_columns.Length == 0)
-//          {
-//             rs = new AddColumnsStream(stream, "fake_col=0");
-//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
-//             rs = new ProjectStream(rs, "!fake_col");
-//          }
-//          else
-//          {
-//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-//          }
-//          if (force_null_into_other_columns)
-//          {
-//             List<string> valid_cols = new List<string>(output_columns);
-//             valid_cols.AddRange(groupby_columns);
-//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-//          }
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
+            ViewStream rs = null;
+            groupby_columns = groupby_columns ?? new string[] { };
+            if (groupby_columns.Length == 0) {
+                // rs = new AddColumnsStream(stream, "fake_col=0");
+                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
+                // rs = new ProjectStream(rs, "!fake_col");
+            } else {
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+            }
+            if (force_null_into_other_columns) {
+                List<string> valid_cols = new List<string>(output_columns);
+                valid_cols.AddRange(groupby_columns);
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+            }
+            mySetRealSource(rs);
+        }
 
-//    public SumStream (ViewStream stream, string outputCol)
-//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
-//    {
-//    }
-// }
+        public SumStream(ViewStream stream, string outputCol)
+        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+        }
+    }
 
-// public class IntSumStream : ViewStream
-// {
-//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//    {
-//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//       mySetRealSource(new GroupByStream(stream,
-//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
-//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//    }
+    public class IntSumStream : ViewStream {
+        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-// }
+    }
 
 
-//    public class SumStream<T> : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
-//       {
-//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
-//       }
-//    }
-// }
+    public class SumStream<T> : ViewStream {
+        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
+            string[] inputCols = spaceSeparatedInputCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -0,0 +1,79 @@
+using BCG;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+using ZstdSharp.Unsafe;
+
+namespace bts {
+    public class RowDiffStream : ViewStream {
+        private ViewStream _vs;
+        private readonly string _order_col, _col_1, _col_2;
+        private readonly int _diff;
+        private IBlueTableSchema _out_schema;
+        private readonly string _diff_col_name;
+        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
+            _vs = vs;
+            _order_col = order_col;
+            _col_1 = col_1;
+            _col_2 = col_2;
+            _diff = diff;
+            _diff_col_name = diff_col_name;
+        }
+
+        private string _diff_col_name_perc {
+            get {
+                return $"{_diff_col_name}_percentage";
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _handle_vs;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _handle_vs;
+        }
+
+        private void _handle_vs(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            IBlueRowCollection nrows = dp.newRows;
+            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
+            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
+            List<IBlueRow> row_list = new List<IBlueRow>();
+            foreach (IBlueRow r in nrows)
+                row_list.Add(r);
+            //OrderBy sorts in ascedning order
+            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
+            var sorted_row_array = sorted_rows.ToArray();
+            // reverse it to make it descending    
+            sorted_rows.Reverse();
+            //sort rows so we can do the diff
+            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
+            int i = 0;
+            foreach (IBlueRow r in sorted_rows) {
+                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
+                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
+                // we assume x - y, and name variables like that
+                double x = r.get<double>(_col_1);
+                int y_row_idx = i + _diff;
+                double diff = System.Double.NaN;
+                double perc = System.Double.NaN;
+                if (y_row_idx < sorted_rows.Count()) {
+                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
+                    diff = x - y;
+                    perc = 100.0 * diff / y;
+                }
+                nr.setValue(_diff_col_name, diff);
+                nr.setValue(_diff_col_name_perc, perc);
+                rv.addRow(nr);
+                i++;
+            }
+
+            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -0,0 +1,34 @@
+using BCG;
+
+namespace bts {
+    public class SchemaGuesserStream : ViewStream {
+
+        private IBlueRowCollection _rows = null;
+        private ViewStream _vs;
+        public SchemaGuesserStream(ViewStream vs) {
+            _vs = vs;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _on_data;
+        }
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _on_data;
+        }
+
+        private void _on_data(object obj) {
+            DataPacket dp = obj as DataPacket;
+            if (_rows == null) {
+                _rows = BlueDataUtility.get_typed_table(dp.newRows);
+            }
+            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+
+
+        }
+
+
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,99 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-namespace BCG;
-
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
-
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
-
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
-
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
-
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
-
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
-
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
-
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace BCG;
+
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
+
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
+
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
+
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
+
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
+
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
+
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
+
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,49 +1,43 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class TopStream : MethodDrivenStream
-//    {
-//       private int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-//       public TopStream (ViewStream stream, 
-//          int topCount, 
-//          string sortExpression) 
-//          :
-//          this (sortExpression == null? 
-//          stream :
-//          new SortStream (stream, sortExpression), topCount)
-//       {}
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-//       protected override void myOnTableClear () 
-//       {
-//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//       }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-//       protected override void myHandleFirstCall () 
-//       {
-//          myLastFiredPacket = new DataPacket ();
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-//       }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-//       protected override void myHandleIncrementalAdd ()     {}
+        protected override void myHandleIncrementalAdd() { }
 
-//       protected override void myHandleIncrementalDelete()   
-//       {
-//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-//             oldRows,
-//             myLastFiredPacket.schema.columnNames,
-//             out myLastFiredPacket.deletedRows,
-//             out myLastFiredPacket.addedRows);
-//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-//       }
-//    }
-// }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using System.IO;
 using System.Diagnostics;
 using System.Linq;
+using System.Globalization;
 
 //using BCG.Utilities;
 using BCG.Utilities.General;
@@ -292,21 +293,118 @@ namespace BCG {
          return TypeCode.Empty;
       }
 
-      public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) {
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         foreach (var col in rows.schema.columnInfo) {
-            var text_vals = BlueDataUtility.get_colvalues<string>(rows, col.colName);
-            TypeCode type_code = convert_column_type(text_vals);
-            cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code]));
+      // public static Type guess_type(string val) {
+      //    //DateTime
+      //    {
+      //       DateTime dt;
+      //       var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //       DateTimeOffset dtoff;
+      //       success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
+      //                                              null, System.Globalization.DateTimeStyles.None, out dtoff);
+
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //    }
+      //    //int
+      //    {
+      //       int i;
+      //       var success = Int32.TryParse(val, out i);
+      //       if (success) {
+      //          return typeof(Int32);
+      //       }
+      //    }
+      //    //double
+      //    {
+      //       double d;
+      //       var success = Double.TryParse(val, out d);
+      //       if (success) {
+      //          return typeof(Double);
+      //       }
+      //    }
+      //    //bool
+      //    {
+      //       bool b;
+      //       var success = Boolean.TryParse(val, out b);
+      //       if (success) {
+      //          return typeof(bool);
+      //       }
+      //    }
+      //    //string
+      //    return typeof(string);
+      // }
+
+      private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
+      public static Type guess_type(string val) {
+         foreach (var t in _parsable_types) {
+            var type_val = convert_to_type(t, val);
+            if (type_val != null)
+               return t;
          }
-         return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         return typeof(string);
+
       }
 
-      // public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) {
-      //    foreach (var ci in out_schema.columnInfo){
+      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
+         // just use one row to guess
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         foreach (IBlueRow r in rows) {
+            foreach (var ci in r.schema.columnInfo) {
+               if (ci.colType != typeof(System.String)) {
+                  cis.Add(ci);
+                  continue;
+               }
+               Type type = guess_type(r[ci.colName] as string);
+               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
+            }
+            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         }
+         return null;
+      }
+
+      static object convert_to_type(Type t, string value) {
+         // Attempt to convert the string to the specified type
+         try {
+            return Convert.ChangeType(value, t);
+         } catch (Exception) {
+            if (t == typeof(double)) {
+               // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
+               string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
+               cleaned_val = cleaned_val.Replace('−', '-');
+               double rv_d;
+               if (Double.TryParse(cleaned_val, out rv_d))
+                  return rv_d;
+               else
+                  return null;
+            }
+            return null;
+            // Handle conversion failure gracefully
+            // throw new ArgumentException("Failed to convert the string to the specified type.");
+         }
+      }
+
+      // converts a row where all columns are string types to a schema where the best types are guesed
+      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
+         Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
+            MutableBlueRow rv = new MutableBlueRow(typed_schema);
+            foreach (ColumnInfo ci in typed_schema.columnInfo) {
+               var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
+               rv.setValue(ci.colName, typed_val);
+            }
+            return rv;
+         };
+         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
+         var rv = new BlueTable(typed_schema);
+         foreach (IBlueRow r in string_rows) {
+            var typed_row = get_typed_row(r, typed_schema);
+            rv.addRow(typed_row);
+         }
+         return rv;
+      }
 
-      //    }
-      // }
 
 
       /// <summary>
@@ -2112,5 +2210,33 @@ namespace BCG {
       }
 
       #endregion
+
+      public class RowComparer<T> : IComparer<IBlueRow> {
+         private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
+         public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
+            _sort_cols = sort_cols;
+         }
+         public int Compare(IBlueRow? x, IBlueRow? y) {
+            Func<object, object, int> compare_cols = (o1, o2) => {
+               return (o1 as IComparable).CompareTo(o2);
+            };
+            foreach (var c in _sort_cols) {
+               int i = compare_cols(x[c.Item1], y[c.Item1]);
+
+            }
+            return 1;
+         }
+
+      }
+      public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
+         List<IBlueRow> row_list = new List<IBlueRow>();
+         foreach (IBlueRow r in rows) {
+            row_list.Add(r);
+         }
+         var row_comparer = new RowComparer<IBlueRow>(sort_cols);
+         ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
+         return row_list;
+
+      }
    }
 }

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -6,7 +6,10 @@ using Bts.DataStreamExpressions;
 using System.Collections.Generic;
 
 using BCG;
-using System.Threading;
+using System.IO;
+using BCG.Utilities.General;
+using bts;
+using System.Linq;
 
 
 bts.Log.Gen.info($"Starting DSPublisher");
@@ -57,6 +60,8 @@ static void start_echo_client() {
 void _publish_streams() {
    PubTableBroker pb_broker = new PubTableBroker();
    CloudStreamManager csm = new CloudStreamManager();
+   string csv_folder = @"C:\Temp\csvs_topublish";
+   CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
    // get it from config
    Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
         //{"bbo","bbo()"},
@@ -65,6 +70,7 @@ void _publish_streams() {
         //{"books","books()"},
        {"seconds_1_10","second(1,10)"},
         {"literal_420","literal(\"x\", \"420.0\")"},
+        {"tradebar_30","tradebars(30)"},
         {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
         };
    //  {"bbo_ts","timeSeries(bbo())"}
@@ -88,4 +94,37 @@ void _publish_streams() {
 
 void on_change(object obj) {
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
+}
+
+public class CSVPublisher : IWorker {
+   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
+   // read from config
+   private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
+
+   private PubTableBroker _pub_table_broker;
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+      _csv_directory = csv_directory;
+      _pub_table_broker = pub_table_broker;
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
+   public void doWork() {
+      var file_names = Directory.GetFiles(_csv_directory);
+      foreach (var fn in file_names) {
+         if (!_pubs.ContainsKey(fn)) {
+            _handle_new_file(fn);
+         }
+      }
+   }
+
+   private void _handle_new_file(string file_name) {
+      if (file_name.EndsWith(".csv")) {
+         ViewStream vs = new CSVReaderStream(file_name);
+         vs = new SchemaGuesserStream(vs);
+         var toks = file_name.Split('\\');
+         string table_name = toks.Last();
+         table_name = table_name.Remove(table_name.Length - ".csv".Length);
+         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
+         _pubs.Add(file_name, vp);
+      }
+   }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs

@@ -0,0 +1,161 @@
+using BCG;
+using BCG.Utilities.General;
+
+using System;
+
+namespace bts {
+    [DataStreamUserRespresentation("tradebars")]
+    public class TradeBarsStream : ViewStream, IWorker {
+
+        public class TradeBar {
+            static double[] calculate_log_returns(double[] prices) {
+                // Calculate logarithmic returns
+                double[] returns = new double[prices.Length - 1];
+                for (int i = 1; i < prices.Length; i++) {
+                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
+                }
+                return returns;
+            }
+
+            static double calculate_volatility(double[] returns) {
+                // Calculate standard deviation of logarithmic returns
+                double mean = returns.Average();
+                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
+                double variance = sum_of_squares / (returns.Length - 1);
+                double volatility = Math.Sqrt(variance);
+
+                return volatility;
+            }
+            public void add_trade(IBlueRow r) {
+                double price = r.get<double>("price");
+                //prices.Add(price);
+                double qty = r.get<double>(col_name: "qty");
+                if (n_entries == 0) {
+                    open = price;
+                    low = price < low ? price : low;
+                    high = price > high ? price : high;
+                }
+                n_entries++;
+                volume += qty;
+                vwap_sum += price * qty;
+                close = price;
+                if ("Buy".Equals(r.get<string>("aggressor"))) {
+                    n_buys_aggs++;
+                } else {
+                    n_sell_aggs++;
+                }
+            }
+            private List<double> prices = new List<double>();
+            public readonly string security;
+            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
+            public uint n_buys_aggs, n_sell_aggs, n_entries;
+            public DateTime bar_start = DateTime.Now;
+            public TradeBar(string security_) {
+                security = security_;
+            }
+            public double volatility {
+                get {
+                    //if (prices.Count == 1)
+                    return 0.0;
+                    var log_returns = calculate_log_returns(prices.ToArray());
+                    return calculate_volatility(log_returns);
+                }
+            }
+        }
+
+        private readonly int _bar_period_seconds;
+        private readonly ViewStreamDatabase _vsd;
+        private ViewStream _timesales_stream;
+        private ViewStream _bbo_stream;
+        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
+        private bool _dirty = false;
+        private TimeSpan _fire_period;
+        private DateTime _last_fire_time = DateTime.Now;
+        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
+        Tuple.Create("id", TypeCode.Int32),
+        Tuple.Create("bar_start", TypeCode.DateTime),
+        Tuple.Create("bar_end", TypeCode.DateTime),
+        Tuple.Create("day_of_week", TypeCode.String),
+        Tuple.Create("n_trades", TypeCode.UInt32),
+        Tuple.Create("security", TypeCode.String),
+        Tuple.Create("sec_key", TypeCode.UInt32),
+        Tuple.Create("low", TypeCode.Double),
+        Tuple.Create("high", TypeCode.Double),
+        Tuple.Create("open", TypeCode.Double),
+        Tuple.Create("close", TypeCode.Double),
+        Tuple.Create("vwap_sum", TypeCode.Double),
+        Tuple.Create("volume", TypeCode.Double),
+        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
+        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
+        Tuple.Create("volatility", TypeCode.Double)));
+
+
+        public TradeBarsStream(int bar_period_seconds)
+        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
+        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
+            _vsd = vsd;
+            _bar_period_seconds = bar_period_seconds;
+            _timesales_stream = new TimeSalesStream(10000, vsd);
+            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
+            _bbo_stream = new BBOStream(_vsd);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _timesales_stream.onChange += _on_timesales;
+            _bbo_stream.onChange += _on_bbo;
+            _vsd.addWorker(this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vsd.removeWorker(this);
+            _timesales_stream.onChange -= _on_timesales;
+            _bbo_stream.onChange -= _on_bbo;
+        }
+
+        public void doWork() {
+            if (DateTime.Now - _last_fire_time > _fire_period) {
+                if (_dirty) {
+                    _dirty = false;
+                    _last_fire_time = DateTime.Now;
+                    BlueTable added_rows = new BlueTable(_table.schema);
+                    foreach (var kv in _sec_bar_map) {
+                        var sec_key = kv.Key;
+                        var tbar = kv.Value;
+                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
+                        tbar.bar_start.DayOfWeek.ToString(),
+                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
+                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
+                                                        tbar.volatility);
+                        added_rows.addRow(new_row);
+                        _table.addRow(new_row);
+                    }
+                    _sec_bar_map.Clear();
+                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
+                    myFireEvent(dp_out);
+                }
+            }
+        }
+
+        private void _on_timesales(object obj) {
+            DataPacket dp = obj as DataPacket;
+            foreach (IBlueRow r in dp.addedRows) {
+                uint sec_key = r.get<UInt32>("sec_key");
+                TradeBar tbar = null;
+                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
+                    tbar = new TradeBar(r.get<string>("sec"));
+                    _sec_bar_map.Add(sec_key, tbar);
+                }
+                _dirty = true;
+                tbar.add_trade(r);
+            }
+
+
+        }
+
+        private void _on_bbo(object obj) {
+
+        }
+    }
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -1,127 +1,130 @@
-namespace BCG {
-   using BCG.Utilities.General;
-
-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
-   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Concurrent;
-   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
-   using System.Linq;
-   using System.Reactive.Linq;
-   using bts.api.core;
-   using Spark.Api.MarketData;
-
-   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
-
-   [DataStreamUserRespresentation("price_stats")]
-   public class PriceStatsStream : ViewStream, IWorker {
-      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
-      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
-      MarketDataClient _md_client;
-      private bool _dirty = false;
-      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("sec_key", TypeCode.Int32),
-                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("open", TypeCode.UInt32),
-                                                Tuple.Create("close", TypeCode.Double),
-                                                Tuple.Create("settlement", TypeCode.Single));
-      IDisposable? subscription;
-
-      // TODO: add cons for symbol, exchange etc 
-      public PriceStatsStream()
-      : this(ViewStreamDatabase.instance) {
-      }
-
-      public PriceStatsStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("PriceStatsStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
-      }
-
-      private void _log_updates(TableUpdate<PriceStats, uint> update) {
-         int i = 0;
-         // foreach (BookTuple bt in update.added) {
-         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-         // foreach (Change<BookTuple> c in update.changes) {
-         //    var bt = c.New;
-         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-      }
-
-      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
-         _log_updates(update);
-         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
-         _cached_state = update.state;
-         _dirty = true;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-         Disposable.dispose(ref subscription);
-      }
-
-      private void _scan() {
-         // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
-         if (_cached_state == null)
-            return;
-         BlueTable new_state = new BlueTable(_pstats_scheam);
-         foreach (var kv in _cached_state) {
-            SecurityKey skey = kv.Key;
-            int id = Convert.ToInt32(skey);
-            PriceStats pstats = kv.Value;
-            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
-            new_state.addRow(r);
-         }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
-         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-            return;
-         myFireEvent(dp);
-      }
-
-      public void doWork() {
-         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
-            _last_poll_time = DateTime.Now;
-            _dirty = false;
-            _scan();
-         }
-      }
-   }
-}
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("price_stats")]
+   public class PriceStatsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private IDisposable _sub = null;
+      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("date", TypeCode.DateTime),
+                                                Tuple.Create("open", TypeCode.Double),
+                                                Tuple.Create("close", TypeCode.Double),
+                                                Tuple.Create("settlement", TypeCode.Double));
+      IDisposable? subscription;
+
+      // TODO: add cons for symbol, exchange etc 
+      public PriceStatsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public PriceStatsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("PriceStatsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
+      }
+
+      private void _log_updates(TableUpdate<PriceStats, uint> update) {
+         int i = 0;
+         // foreach (BookTuple bt in update.added) {
+         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+         // foreach (Change<BookTuple> c in update.changes) {
+         //    var bt = c.New;
+         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+      }
+
+      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+         _dirty = true;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+         Disposable.dispose(ref _sub);
+      }
+
+      private void _scan() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_pstats_scheam);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            PriceStats pstats = kv.Value;
+            var today = DateTime.Now.Date;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoDataVM.cs

@@ -1,39 +1,42 @@
-using bts.api.orderman;
-using bts.api.orderman.algo;
-using bts.utils;
-
-using mbts.spark.utilities;
-
-using System;
-using System.Windows.Media;
-
-namespace mbts.spark {
-   struct AlgoDataVM : IEquatable<AlgoDataVM> {
-      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
-
-      public readonly AlgoData data;
-      public readonly AlgoDefinitionMetadata? meta;
-
-      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
-         this.data = data;
-         this.meta = meta;
-      }
-
-      public string name => data.name;
-      public Brush badge_fill => OrderUtils.badge_fill(meta, OrderDataHelper.get_metadata(data.value.CustomCase));
-      public AlgoDisplayGroup group => data.key.display_group();
-
-      public override string ToString() =>
-         data.name;
-
-      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
-         new AlgoDataVM(data, meta);
-
-      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
-         new AlgoDataVM(tup.algo, tup.meta);
-
-      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
-      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
-      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
-   }
-}
+using bts.api.orderman;
+using bts.api.orderman.algo;
+using bts.utils;
+
+using mbts.spark.utilities;
+
+using System;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   struct AlgoDataVM : IEquatable<AlgoDataVM> {
+      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
+
+      public readonly AlgoData data;
+      public readonly AlgoDefinitionMetadata? meta;
+
+      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
+         this.data = data;
+         this.meta = meta;
+      }
+
+      public string name => data.name;
+
+      public Brush badge_fill => OrderUtils.badge_fill(
+         meta, OrderMetadataStore.get_metadata(data.value.CustomCase));
+
+      public AlgoDisplayGroup group => data.key.display_group();
+
+      public override string ToString() =>
+         data.name;
+
+      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
+         new AlgoDataVM(data, meta);
+
+      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
+         new AlgoDataVM(tup.algo, tup.meta);
+
+      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
+      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
+   }
+}

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -298,7 +298,7 @@ namespace mbts.spark {
          }
 
          bool has_algo_params(OrderSecurityAlgo? order) =>
-            order.HasValue && !OrderDataHelper.get_metadata(order.Value.algo.value.CustomCase).NoProps;
+            order.HasValue && !OrderMetadataStore.get_metadata(order.Value.algo.value.CustomCase).NoProps;
 
          Window mk_algo_params_dialog(OrderSecurityAlgo? order) {
             if (!has_algo_params(order))

src/client/spark/Orders/OrderFactory.cs

@@ -1,323 +1,323 @@
-#nullable enable
-namespace mbts.spark.utilities {
-   using bts;
-   using bts.api;
-   using bts.api.core;
-   using bts.api.orderman;
-   using bts.collections;
-   using bts.spark;
-
-   using mbts.ui;
-   using mbts.ui.utilities;
-   using mbts.ui.utility;
-   using mbts.utils;
-
-   using Spark.Core;
-   using Spark.Core.Disposables;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Composition;
-   using System.Linq;
-   using System.Reactive.Linq;
-
-   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
-   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
-
-   [Export(typeof(OrderFactory))]
-   class OrderFactory : IDisposable {
-      static readonly OrderMetadata _default_order_metadata = new() {
-         KeyBinding = "",
-         Color = BTSColors.Orders.default_order
-      };
-      readonly AdminModel _admin_model;
-      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
-      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
-
-      IDisposable? _dispose;
-
-      [ImportingConstructor]
-      public OrderFactory(OrderService service, AdminModel admin_model) {
-         _admin_model = admin_model;
-         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
-            .DynamicOrderDesc
-            .ToDictionary(d => d.DynamicOrderType);
-         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
-            OneOf<OrderType, string>.ValueEqualityComparer);
-         // add standard types
-         algo_name_index.AddRange(
-            EnumHelper.get_values_except(OrderType.None)
-            .Select(t => new OneOf<OrderType, string>(t)));
-         // add dynamic types
-         algo_name_index.AddRange(_dynamic_descriptors.Keys
-            .Select(dt => new OneOf<OrderType, string>(dt)));
-         // build abbreviation index
-         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
-            algo_name_index,
-            oneof => oneof.value.ToString(),
-            OneOf<OrderType, string>.ValueEqualityComparer);
-      }
-
-      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
-         if (dp is null) {
-            return null;
-         }
-         return dp.ValueCase switch {
-            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
-            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
-            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
-            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
-            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
-            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
-            DynamicOrderParamType.AsString => dp.AsString,
-            _ => $"({dp.ValueCase})",
-         };
-      }
-
-      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
-         var rv = new DynamicOrderParam();
-         switch (descriptor.ParamType.DescriptorCase) {
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
-               rv.AsArray = new DynamicOrderArray();
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
-               rv.AsEnumerated = new() {
-                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
-               };
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
-               switch (descriptor.ParamType.Primitive) {
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
-               }
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
-               rv.AsStruct = new();
-               break;
-         }
-         return rv;
-      }
-
-      public static SpreaderOrderDefinition spreader_def() => new() {
-         ExecutionStrategy = new ExecutionStrategy()
-      };
-
-      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
-         _dynamic_descriptors.GetValueOrDefault(type);
-
-      public DynamicOrderDefinition? dynamic_def(string type) {
-         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
-            return null;
-         }
-         var def = new DynamicOrderDefinition { DynamicOrderType = type };
-         foreach (var p in desc.ParamDesc) {
-            def.OrderParams[p.ParamName] = init_dynamic_param(p);
-         }
-         return def;
-      }
-
-      public OrderType default_order_type(SecurityImmutable security) =>
-         security.ContractCase switch {
-            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
-            _ => OrderType.Limit,
-         };
-
-      public TimeInForce default_time_in_force() =>
-         TimeInForce.Day;
-
-      public TimeInForce default_time_in_force(SecurityImmutable security) =>
-         default_time_in_force();
-
-      public IEnumerable<OrderType> default_order_types() =>
-         OrderDataHelper.exchange_order_types;
-
-      public IEnumerable<TimeInForce> default_time_in_forces(
-         SecurityImmutable security,
-         OrderType order_type,
-         bool is_child = false
-      ) {
-         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
-         if (order_type != OrderType.Limit) {
-            tifs = tifs.Except(new[] { TimeInForce.Ioc });
-         }
-         return tifs;
-      }
-
-      public IEnumerable<string> dynamic_types() =>
-         _dynamic_descriptors.Keys;
-
-      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
-         if (Enum.TryParse(type_name, out OrderType sys_type)) {
-            return _custom_def(sys_type);
-         }
-         var dyn_def = dynamic_def(type_name);
-         if (dyn_def is null) {
-            return null;
-         }
-         return new(new() { Dynamic = dyn_def });
-      }
-
-      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
-         var rv = new OrderDefinition {
-            SecurityKey = security.Id,
-            Side = Side.Buy,
-            TimeInForce = TimeInForce.Day,
-            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
-            CtiCode = CTICode.CtiCode1,
-            CustomerOrFirm = CustomerOrFirm.Firm,
-            // this will be the tag 50 on CME orders
-            // current user might be okay, but we might also want to get it
-            // from the store that serves the risk admin gui
-            ManualOrderIndicator = ManualOrderIndicator.Yes,
-            User = _admin_model.user.name,
-            UserId = _admin_model.user.id
-         };
-         if (account.HasValue) {
-            rv.Account = account.Value.name;
-            rv.AccountId = account.Value.id;
-         }
-         return rv;
-      }
-
-      public string abbreviate_algo(OneOf<OrderType, string> id) {
-         if (id is null) {
-            throw new ArgumentNullException(nameof(id));
-         }
-         if (id.value is string) {
-            return _algo_abbreviation_index.GetValueOrDefault(id) ??
-               id.value.ToString() ??
-               string.Empty;
-         }
-         if (id.value is OrderType t) {
-            return OrderDataHelper.abbreviate_algo_type(t);
-         }
-         return id.ToString() ?? string.Empty;
-      }
-
-      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
-         get_metadata(order.Def.CustomDef.CustomCase);
-
-      public OrderMetadata get_metadata(OrderType order_type) =>
-         order_type switch {
-            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
-            _ => _default_order_metadata
-         };
-
-      public SpreaderOrderDefinition spreader_order(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new() {
-            Security = security.CloneToMutable(),
-            ExecutionStrategy = strategy.CloneToMutable()
-         };
-
-      public SpreaderOrderDefinitionImmutable spreader_orderi(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new(spreader_order(security, strategy));
-
-      public void Dispose() =>
-         Disposable.dispose(ref _dispose);
-
-      static CustomOrderDefinition _custom_def() =>
-         new() { Limit = _limit_def() };
-
-      static CustomOrderDefinition _custom_def(
-         OrderType type) {
-         var rv = new CustomOrderDefinition();
-         switch (type) {
-            case OrderType.Market: rv.Market = _market_def(); break;
-            case OrderType.Limit: rv.Limit = _limit_def(); break;
-            case OrderType.Stop: rv.Stop = _stop_def(); break;
-            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
-            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
-            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
-            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
-            case OrderType.Tick: rv.Tick = _tick_def(); break;
-            case OrderType.Lean: rv.Lean = _lean_def(); break;
-            case OrderType.Join: rv.Join = _join_def(); break;
-            case OrderType.Top: rv.Top = _top_def(); break;
-            case OrderType.Cover: rv.Cover = _cover_def(); break;
-            case OrderType.Oco: rv.Oco = _oco_def(); break;
-            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
-            case OrderType.Offset: rv.Offset = _offset_def(); break;
-            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
-            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
-            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
-            default:
-               OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, type);
-               break;
-         }
-         return rv;
-      }
-
-      static MarketOrderDefinition _market_def() => new();
-      static LimitOrderDefinition _limit_def() => new();
-      static StopOrderDefinition _stop_def() => new();
-      static StopLimitOrderDefinition _stop_limit_def() => new();
-      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
-      static IcebergOrderDefinition _iceberg_def() =>
-         new() { Child = _custom_def(), ShowQuantity = 1 };
-      static TimeBoxOrderDefinition _time_box_def() => new() {
-         Child = _custom_def(),
-         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
-         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
-      };
-      static TickOrderDefinition _tick_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 }
-      };
-      static LeanOrderDefinition _lean_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 },
-         MaxChaseTicks = 2
-      };
-      static JoinOrderDefinition _join_def() => new() {
-         Child = _custom_def(),
-         JoinThreshold = new() { Absolute = 1 }
-      };
-      static TopOrderDefinition _top_def() => new() {
-         Child = _custom_def(),
-         TopThreshold = new() { Absolute = 1 }
-      };
-      static CoverOrderDefinition _cover_def() => new() {
-         Child = _custom_def(),
-         Hedge = _custom_def(),
-         HedgeTicks = 1
-      };
-      static OCOOrderDefinition _oco_def() => new() {
-         Child = _custom_def(),
-         Other = _custom_def(),
-         OtherSide = RelativeSide.Same
-      };
-      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
-      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
-      static TriggerOrderDefinition _trigger_def() => new() {
-         Child = _custom_def(),
-         TriggerCondition = new TriggerCondition(),
-         TriggerAction = new TriggerAction(),
-      };
-      static DynamicOrderDefinition _dynamic_def() => new();
-   }
-}
+#nullable enable
+namespace mbts.spark.utilities {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.collections;
+   using bts.spark;
+
+   using mbts.ui;
+   using mbts.ui.utilities;
+   using mbts.ui.utility;
+   using mbts.utils;
+
+   using Spark.Core;
+   using Spark.Core.Disposables;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Composition;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
+   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
+
+   [Export(typeof(OrderFactory))]
+   class OrderFactory : IDisposable {
+      static readonly OrderMetadata _default_order_metadata = new() {
+         KeyBinding = "",
+         Color = BTSColors.Orders.default_order
+      };
+      readonly AdminModel _admin_model;
+      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
+      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
+
+      IDisposable? _dispose;
+
+      [ImportingConstructor]
+      public OrderFactory(OrderService service, AdminModel admin_model) {
+         _admin_model = admin_model;
+         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
+            .DynamicOrderDesc
+            .ToDictionary(d => d.DynamicOrderType);
+         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
+            OneOf<OrderType, string>.ValueEqualityComparer);
+         // add standard types
+         algo_name_index.AddRange(
+            EnumHelper.get_values_except(OrderType.None)
+            .Select(t => new OneOf<OrderType, string>(t)));
+         // add dynamic types
+         algo_name_index.AddRange(_dynamic_descriptors.Keys
+            .Select(dt => new OneOf<OrderType, string>(dt)));
+         // build abbreviation index
+         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
+            algo_name_index,
+            oneof => oneof.value.ToString(),
+            OneOf<OrderType, string>.ValueEqualityComparer);
+      }
+
+      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
+         if (dp is null) {
+            return null;
+         }
+         return dp.ValueCase switch {
+            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
+            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
+            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
+            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
+            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
+            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
+            DynamicOrderParamType.AsString => dp.AsString,
+            _ => $"({dp.ValueCase})",
+         };
+      }
+
+      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
+         var rv = new DynamicOrderParam();
+         switch (descriptor.ParamType.DescriptorCase) {
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
+               rv.AsArray = new DynamicOrderArray();
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
+               rv.AsEnumerated = new() {
+                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
+               };
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+               switch (descriptor.ParamType.Primitive) {
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
+               }
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
+               rv.AsStruct = new();
+               break;
+         }
+         return rv;
+      }
+
+      public static SpreaderOrderDefinition spreader_def() => new() {
+         ExecutionStrategy = new ExecutionStrategy()
+      };
+
+      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
+         _dynamic_descriptors.GetValueOrDefault(type);
+
+      public DynamicOrderDefinition? dynamic_def(string type) {
+         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
+            return null;
+         }
+         var def = new DynamicOrderDefinition { DynamicOrderType = type };
+         foreach (var p in desc.ParamDesc) {
+            def.OrderParams[p.ParamName] = init_dynamic_param(p);
+         }
+         return def;
+      }
+
+      public OrderType default_order_type(SecurityImmutable security) =>
+         security.ContractCase switch {
+            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
+            _ => OrderType.Limit,
+         };
+
+      public TimeInForce default_time_in_force() =>
+         TimeInForce.Day;
+
+      public TimeInForce default_time_in_force(SecurityImmutable security) =>
+         default_time_in_force();
+
+      public IEnumerable<OrderType> default_order_types() =>
+         OrderDataHelper.exchange_order_types;
+
+      public IEnumerable<TimeInForce> default_time_in_forces(
+         SecurityImmutable security,
+         OrderType order_type,
+         bool is_child = false
+      ) {
+         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
+         if (order_type != OrderType.Limit) {
+            tifs = tifs.Except(new[] { TimeInForce.Ioc });
+         }
+         return tifs;
+      }
+
+      public IEnumerable<string> dynamic_types() =>
+         _dynamic_descriptors.Keys;
+
+      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
+         if (Enum.TryParse(type_name, out OrderType sys_type)) {
+            return _custom_def(sys_type);
+         }
+         var dyn_def = dynamic_def(type_name);
+         if (dyn_def is null) {
+            return null;
+         }
+         return new(new() { Dynamic = dyn_def });
+      }
+
+      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
+         var rv = new OrderDefinition {
+            SecurityKey = security.Id,
+            Side = Side.Buy,
+            TimeInForce = TimeInForce.Day,
+            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
+            CtiCode = CTICode.CtiCode1,
+            CustomerOrFirm = CustomerOrFirm.Firm,
+            // this will be the tag 50 on CME orders
+            // current user might be okay, but we might also want to get it
+            // from the store that serves the risk admin gui
+            ManualOrderIndicator = ManualOrderIndicator.Yes,
+            User = _admin_model.user.name,
+            UserId = _admin_model.user.id
+         };
+         if (account.HasValue) {
+            rv.Account = account.Value.name;
+            rv.AccountId = account.Value.id;
+         }
+         return rv;
+      }
+
+      public string abbreviate_algo(OneOf<OrderType, string> id) {
+         if (id is null) {
+            throw new ArgumentNullException(nameof(id));
+         }
+         if (id.value is string) {
+            return _algo_abbreviation_index.GetValueOrDefault(id) ??
+               id.value.ToString() ??
+               string.Empty;
+         }
+         if (id.value is OrderType t) {
+            return OrderDataHelper.abbreviate_algo_type(t);
+         }
+         return id.ToString() ?? string.Empty;
+      }
+
+      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
+         get_metadata(order.Def.CustomDef.CustomCase);
+
+      public OrderMetadata get_metadata(OrderType order_type) =>
+         order_type switch {
+            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
+            _ => _default_order_metadata
+         };
+
+      public SpreaderOrderDefinition spreader_order(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new() {
+            Security = security.CloneToMutable(),
+            ExecutionStrategy = strategy.CloneToMutable()
+         };
+
+      public SpreaderOrderDefinitionImmutable spreader_orderi(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new(spreader_order(security, strategy));
+
+      public void Dispose() =>
+         Disposable.dispose(ref _dispose);
+
+      static CustomOrderDefinition _custom_def() =>
+         new() { Limit = _limit_def() };
+
+      static CustomOrderDefinition _custom_def(
+         OrderType type) {
+         var rv = new CustomOrderDefinition();
+         switch (type) {
+            case OrderType.Market: rv.Market = _market_def(); break;
+            case OrderType.Limit: rv.Limit = _limit_def(); break;
+            case OrderType.Stop: rv.Stop = _stop_def(); break;
+            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
+            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
+            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
+            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
+            case OrderType.Tick: rv.Tick = _tick_def(); break;
+            case OrderType.Lean: rv.Lean = _lean_def(); break;
+            case OrderType.Join: rv.Join = _join_def(); break;
+            case OrderType.Top: rv.Top = _top_def(); break;
+            case OrderType.Cover: rv.Cover = _cover_def(); break;
+            case OrderType.Oco: rv.Oco = _oco_def(); break;
+            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
+            case OrderType.Offset: rv.Offset = _offset_def(); break;
+            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
+            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
+            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
+            default:
+               OrderDataHelper.setup_generic_custom_order(rv, type);
+               break;
+         }
+         return rv;
+      }
+
+      static MarketOrderDefinition _market_def() => new();
+      static LimitOrderDefinition _limit_def() => new();
+      static StopOrderDefinition _stop_def() => new();
+      static StopLimitOrderDefinition _stop_limit_def() => new();
+      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
+      static IcebergOrderDefinition _iceberg_def() =>
+         new() { Child = _custom_def(), ShowQuantity = 1 };
+      static TimeBoxOrderDefinition _time_box_def() => new() {
+         Child = _custom_def(),
+         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
+         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
+      };
+      static TickOrderDefinition _tick_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 }
+      };
+      static LeanOrderDefinition _lean_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 },
+         MaxChaseTicks = 2
+      };
+      static JoinOrderDefinition _join_def() => new() {
+         Child = _custom_def(),
+         JoinThreshold = new() { Absolute = 1 }
+      };
+      static TopOrderDefinition _top_def() => new() {
+         Child = _custom_def(),
+         TopThreshold = new() { Absolute = 1 }
+      };
+      static CoverOrderDefinition _cover_def() => new() {
+         Child = _custom_def(),
+         Hedge = _custom_def(),
+         HedgeTicks = 1
+      };
+      static OCOOrderDefinition _oco_def() => new() {
+         Child = _custom_def(),
+         Other = _custom_def(),
+         OtherSide = RelativeSide.Same
+      };
+      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
+      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
+      static TriggerOrderDefinition _trigger_def() => new() {
+         Child = _custom_def(),
+         TriggerCondition = new TriggerCondition(),
+         TriggerAction = new TriggerAction(),
+      };
+      static DynamicOrderDefinition _dynamic_def() => new();
+   }
+}

src/client/spark/Orders/OrderTypeVMs/DefaultAlgoVM.cs

@@ -35,7 +35,7 @@ namespace mbts.spark {
       public IAlgoVM impl { get; private set; }
       public AlgoKey id { get; } = AlgoKey.from_known_type(AlgoType.None);
       public AlgoType order_type => _definition.value.CustomCase;
-      public OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+      public OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
       public bool can_expand => impl?.can_expand ?? false;
       public bool is_expanded {
          get => false;

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -41,17 +41,6 @@ namespace mbts.spark {
             object vm = Activator.CreateInstance(makevm, key, algo_type, display_name, editing_permissions);
             return (IAlgoVM)vm;
          }
-         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
-            var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
-            var rt = prop.PropertyType;
-            var rtt = Activator.CreateInstance(rt);
-
-            var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
-            foreach (var cid in ci)
-               cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
-            prop.SetValue(custom, rtt);
-         }
       }
 
       class GenericOrderVM<TMutable, TImmutable> : OrderTypeVMBase<TMutable, TImmutable>

src/client/spark/Orders/OrderTypeVMs/OrderMetadataStore.cs

@@ -0,0 +1,32 @@
+namespace mbts.spark;
+
+using bts.api.orderman;
+using bts.spark;
+
+using mbts.ui;
+
+static class OrderMetadataStore {
+   public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
+      order_type switch {
+         CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+         CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+         CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+         CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+         CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+         CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+         CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+         CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+         CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+         CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+         CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+         CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+         CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
+         _ => new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order },
+      };
+
+}

src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs

@@ -201,7 +201,7 @@ namespace mbts.spark {
          public virtual string abbreviation => model == null ? display_name : model.factory.abbreviate_algo(new OneOf<AlgoType, string>(order_type));
          public virtual AlgoDisplayGroup group => order_type.is_bts_algo_type() ? AlgoDisplayGroup.BTS : AlgoDisplayGroup.Exchange;
          public OrdersModel orders_model => model;
-         public virtual OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+         public virtual OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
          // child algos
          [EditorBrowsable(EditorBrowsableState.Never)]
          public dynamic children_vm {

src/client/spark_api/MarketData/BookTuple.cs

@@ -36,7 +36,7 @@ namespace Spark.Api.MarketData {
       public override int GetHashCode() =>
          security.Id.GetHashCode();
       public static Object ToJsonObj(BookTuple obj) => new {
-         secuirty = obj.security.ToString(),
+         security = obj.security.ToString(),
          book = obj.book.ToJsonObj()
       };
       public static bool operator ==(BookTuple left, BookTuple right) =>

src/client/spark_api/MarketData/Co2NativeMethods.cs

@@ -67,7 +67,10 @@ namespace mbts.co2 {
       [DllImport("libspark")]
       internal unsafe static extern int md_consumer_read_trades(
          IntPtr consumer, UserSymbol channel, ref long cursor, raw_receive_pb cb);
-
+      [DllImport("libspark")]
+      internal unsafe static extern int md_consumer_read_trades_linux(
+         IntPtr consumer, string symbol, int venue, int type, ref long cursor, raw_receive_pb cb);
+         
       [DllImport("libspark")]
       internal unsafe static extern int md_consumer_read_rfqs(
          IntPtr consumer, UserSymbol channel, ref long cursor, raw_receive_pb cb);

src/client/spark_api/MarketData/MDConsumerChannel.cs

@@ -23,7 +23,7 @@ namespace mbts.co2 {
 
       public unsafe IList<TradeMessage> read_trades(ref long cursor) {
          IList<TradeMessage> trades = Array.Empty<TradeMessage>();
-         Co2NativeMethods.md_consumer_read_trades(_consumer.Handle, _symbol, ref cursor, callback);
+         Co2NativeMethods.md_consumer_read_trades_linux(_consumer.Handle, _symbol.symbol,(int)_symbol.exchange,(int)_symbol.type , ref cursor, callback);
          return trades;
 
          void callback(byte* buffer, ulong size) =>

src/client/spark_api/MarketData/TradeInfo.cs

@@ -28,7 +28,7 @@ namespace mbts.spark {
       public TickDisplaySettingsImmutable qty_display => security_md.quantity_display;
 
       public static System.Object ToJsonObj(TradeInfo obj) => new {
-         secuirty = obj.security.Exchange.ToString() + "." + obj.security.ToString(),
+         security = obj.security.Exchange.ToString() + "." + obj.security.ToString(),
          price = obj.trade.price(),
          timestamp = obj.trade.Time,
          quantity = obj.volume,

src/client/spark_api/Orders/OrderDataHelper.cs

@@ -1,5 +1,4 @@
-using bts;
-using bts.api;
+using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
@@ -10,23 +9,19 @@ using Google.Protobuf;
 using Google.Protobuf.Reflection;
 using Google.Protobuf.WellKnownTypes;
 
-using mbts.ui;
-using mbts.ui.utilities;
 using mbts.ui.utility;
 using mbts.utils;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.Composition;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Reflection;
 
 using OrderState = bts.api.orderman.OrderState;
 
 namespace mbts.spark.utilities {
-   struct ChildAlgoData {
+   public readonly struct ChildAlgoData {
       public readonly string name;
       public readonly Func<IMessage, CustomOrderDefinition> get_child_from_impl;
       public readonly Action<IMessage, CustomOrderDefinition> assign_child_to_impl;
@@ -48,27 +43,29 @@ namespace mbts.spark.utilities {
       }
    }
 
-   static class OrderDataHelper {
-      class AlgoTypeCaseDescriptorData {
-         public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
-         public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
-         public readonly FieldDescriptor type_case_descriptor;
+   public class AlgoTypeCaseDescriptorData {
+      public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
+      public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
+      public readonly FieldDescriptor type_case_descriptor;
 
-         public AlgoTypeCaseDescriptorData(FieldDescriptor type_case_descriptor, IReadOnlyList<ChildAlgoData> child_algo_data, IReadOnlyList<FieldDescriptor> value_descriptors) {
-            this.type_case_descriptor = type_case_descriptor;
-            this.child_algo_data = child_algo_data;
-            this.value_descriptors = value_descriptors;
-         }
+      public AlgoTypeCaseDescriptorData(
+         FieldDescriptor type_case_descriptor,
+         IReadOnlyList<ChildAlgoData> child_algo_data,
+         IReadOnlyList<FieldDescriptor> value_descriptors
+      ) {
+         this.type_case_descriptor = type_case_descriptor;
+         this.child_algo_data = child_algo_data;
+         this.value_descriptors = value_descriptors;
       }
+   }
 
+   public static class OrderDataHelper {
       const string _primary_child_field = "child";
       // algo-type-case maps
       // Note: obv this will break if we change the name of the 'CustomOneofCase' field
       static readonly OneofDescriptor _oneof_desc = CustomOrderDefinition.Descriptor.Oneofs.FirstOrDefault(fd => fd.Name == "custom");
       static readonly IReadOnlyDictionary<System.Type, CustomOrderDefinition.CustomOneofCase> _algo_type_case_index;
       static readonly IReadOnlyDictionary<CustomOrderDefinition.CustomOneofCase, AlgoTypeCaseDescriptorData> _algo_type_case_descriptor_data;
-      // 
-      static readonly OrderMetadata _default_order_metadata = new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order };
 
       static OrderDataHelper() {
          var algo_type_case_index = new Dictionary<System.Type, CustomOrderDefinition.CustomOneofCase>();
@@ -106,6 +103,12 @@ namespace mbts.spark.utilities {
       }
 
       #region Algo Structure/Hierarchy
+      public static AlgoTypeCaseDescriptorData get_algo_type_descriptor(
+         CustomOrderDefinition.CustomOneofCase algo_type) =>
+         _algo_type_case_descriptor_data.TryGetValue(algo_type, out var d) ? d :
+            throw new ArgumentException(
+               $"Failed to find descriptor data for algo type '{algo_type}'", nameof(algo_type));
+
       public static IEnumerable<(CustomOrderDefinition.CustomOneofCase type, string dyn, uint studio, uint spreader_sec)> enumerate_algo_structure(CustomOrderDefinitionImmutable custom_defi) {
          string dyn = null;
          uint studio = 0, spreader_sec = 0;
@@ -205,29 +208,6 @@ namespace mbts.spark.utilities {
          _algo_type_case_descriptor_data.TryGetValue(algo_type, out AlgoTypeCaseDescriptorData d) ? d.child_algo_data : Array.Empty<ChildAlgoData>();
       #endregion
 
-      public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
-         order_type switch {
-            CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
-            _ => _default_order_metadata,
-         };
-
       public static string abbreviate_algo_type(CustomOrderDefinition.CustomOneofCase algo_type) {
          switch (algo_type) {
             case CustomOrderDefinition.CustomOneofCase.Limit: return "LM";
@@ -331,6 +311,19 @@ namespace mbts.spark.utilities {
          order_type_def != null && _algo_type_case_index.TryGetValue(order_type_def.Descriptor.ClrType, out CustomOrderDefinition.CustomOneofCase algo_case) ?
             algo_case : CustomOrderDefinition.CustomOneofCase.None;
 
+      public static void setup_generic_custom_order(
+         CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
+         var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
+            BindingFlags.Instance | BindingFlags.Public);
+         var rt = prop.PropertyType;
+         var rtt = Activator.CreateInstance(rt);
+
+         var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
+         foreach (var cid in ci)
+            cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
+         prop.SetValue(custom, rtt);
+      }
+
       #region Private Members
       // get child
       static CustomOrderDefinitionImmutable? _get_iceberg_child(IcebergOrderDefinitionImmutable iceberg) => iceberg.HasChild ? iceberg.Child : (CustomOrderDefinitionImmutable?)null;
@@ -655,7 +648,7 @@ namespace mbts.spark.utilities {
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = SpreaderOrderDefinition(); break;
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Dynamic: rv.Dynamic = DynamicOrderDefinition(); break;
                default:
-                  OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, order_type);
+                  setup_generic_custom_order(rv, order_type);
                   break;
             }
             return rv;

src/client/spark_api_server/Program.cs

@@ -38,9 +38,9 @@
          var gwy = new bts.websocket_core.WebsocketConnectionGateway();
          var prms = new WebsocketServerConfig {
             DefaultEndPoint = "/",
-            Host = "titan",
+            Host = "0.0.0.0",
             Scheme = "http",
-            Port = "8080",
+            Port = "3000",
             KeepAliveIntervalSecs = 0.1,
          };
          gwy.init(sync);

src/client/spark_websocket/Core/WebsocketConnectionGateway.cs

@@ -424,6 +424,7 @@ namespace bts.websocket_core {
       /// </summary>
       /// <param name="server_params">The parameters for the websocket server includind the URL and endpoint,</param>
       public Task start_listening(WebsocketServerConfig server_params) {
+         LoggerContext logger = new(Log.Gen, $"WebsocketGateway.start_listening ", "");
          // start listening, assert that init has been called already
          bts.Log.notnull(_sync);
          bts.Log.isnull(_server_params);
@@ -433,6 +434,7 @@ namespace bts.websocket_core {
          _server_params = server_params;
          var builder = WebApplication.CreateBuilder();
          builder.WebHost.UseUrls(_server_params.Uri());
+         logger.warn(_server_params.Uri());
          // var MyAllowSpecificOrigins = "_myAllowSpecificOrigins";
          builder.Services.AddLogging(builder => {
             builder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)

src/client/spark_websocket/Core/WebsocketDataSync.cs

@@ -181,7 +181,7 @@ namespace bts.websocket_core {
       public void init(WebsocketConnectionGateway gtwy) {
          Gateway = gtwy;
          clients = new ClientsStore();
-         TimeSpan md_throttle = new TimeSpan(10000);
+         TimeSpan md_throttle = TimeSpan.FromSeconds(0.1);
          MD_Source = new MDSource(md_throttle);
          Md_Client = new(MD_Source);
          _securities_sub = MD_Source.subscribe_securities().Subscribe(sec_map => _securities = sec_map);

src/client/spark_websocket/Core/WebsocketRestHandler.cs

@@ -160,7 +160,7 @@ namespace bts.websocket_core {
          user_id = (uint)user.Id;
          BTSAccount? account = admin.Accounts.FirstOrDefault(x => x.Name.ToLowerInvariant() == accountname.ToLowerInvariant());
          if (account == null) {
-            return (false, null, "account not corrent");
+            return (false, null, "account not correct");
          }
          account_id = (uint)account.Id;
          if (!WebsocketJsonParsers.TryParse<Side>(

src/data_files/documents/order_flow.txt

@@ -79,6 +79,24 @@ Misc:
 To search for a publisher for an event search for make_stream<Event_Name>, for 
 example make_stream<OrderStateUpdateEvent>()
 
+is_gateway_connected can be false in the NativeOrder class (GlbxNativeOrder) ->
+OrderServices->is_gateway_connected->
+SessionServices->is_gateway_connected
+SessionServicesData->gw_connected
+OrderManager sets SessionServicesData->gw_connected on void OrderManager::on_gateway_state(const mercury::Update<GatewayStatusChangeEvent> &m)
+
+
+Exchange gateway is down (debugging):
+-------------------------------------
+This happens at times and if often a config issue.
+GlbCNativeOrder::_on_activate ->
+OrderServices::is_gateway_connected ->
+SessionServices::is_gateway_connected ->
+SessionServicesData::
+
+IExecutionGateway::on_gw_status_change is what sends out the status
+
+
 --------------------------------------------------------------------------
 market data
 --------------------------------------------------------------------------

src/lib/api/md/GlbxMarketDataConfig.proto

@@ -6,17 +6,19 @@ package bts.api.md;
 
 option csharp_namespace = "bts.api.md";
 
+message GlbxActivePlatform {
+  string host = 1;
+  string user = 2;
+  string password = 3;
+}
+message GlbxOnixPlatform {
+  string temp =1; // will add everything related to onix here.
+}
 
 message GlbxMarketDataConfig {
-  uint32 unique_check_value = 1;
-  //uint32 num_depth_levels = 1;
-  //uint32 depth_throttle_ms = 2;
-  bts.api.domain.SecurityCriteria security_criteria = 3;
-  // Market data isn't subscribed for securities listed here (even if they are
-  // included by 'security_criteria'). There are some securities that don't
-  // fit into our system just yet, such as 'BTCBIDR', which trades in prices
-  // larger than our cpp 'domain::Price' class supports.
-  //repeated string excluded_securities = 4;
-  //string url_rest_base = 5;
-  //string url_ws_base = 6;
+  bts.api.domain.SecurityCriteria security_criteria = 2;
+  oneof Platform {
+    GlbxActivePlatform glbx_active_platform = 10;
+    GlbxOnixPlatform glbx_onix_platform = 11;
+  }
 }

src/lib/cmake/GetLinuxServerLinkLibraries.cmake

@@ -25,8 +25,23 @@ function(get_link_libraries dest)
     Packet++
     Common++
     ssh2
+    ACE-5.7.5
+    ActivBase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivOneApi_x86-64_ubuntu22-04_gcc123_mdid
+    ActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid
+    ActivDatabase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivFramework_x86-64_ubuntu22-04_gcc123_mdid
+    ActivServices_x86-64_ubuntu22-04_gcc123_mdid
+    BoostAtomic_x86-64_ubuntu22-04_gcc123_mdid
+    BoostChrono_x86-64_ubuntu22-04_gcc123_mdid
+    BoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid
+    BoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid
+    -lm
+    -lgcc_s
+    -lc
     -lrt
     -ldl
+    -lstdc++
     -lpcap
     -static-libgcc
     -static-libstdc++

src/lib/config/profiles/default/jetfix.conf

@@ -3,7 +3,7 @@ JetFIXSimConnection {
    value {
       sim {
         name: "sim"
-        venue: VENUE_SIM
+        venue: VENUE_GLBX
         socket_connect_host_ip: "127.0.0.1"
         socket_connect_port: @Ports.glbx_sim_fix_port
         sender_compid: "SPARK" # this is the FIX session name

src/lib/core/TimeStamp.cpp

@@ -23,7 +23,18 @@ TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
    value_ =
        std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
 }
+bool TimeStamp::from_ymd(
+    std::string yearMonthDay, TimeStamp *result) {
+   uint32_t year, month, day;
+   std::istringstream ss(yearMonthDay);
+    char dash; // to consume the dash character in the input
 
+    // Use stringstream to extract year, month, and day from the input string
+    ss >> year >> dash >> month >> dash >> day;
+    if(ss.fail())
+      return false;
+    return TimeStamp::from_ymd(year,month,day,result);
+}
 bool TimeStamp::from_ymd(
     std::uint32_t year, std::uint32_t month, std::uint32_t day, TimeStamp *result) {
    if (!is_valid_ymd(year, month, day)) {

src/lib/core/TimeStamp.h

@@ -41,6 +41,9 @@ struct TimeStamp {
        std::uint32_t month,
        std::uint32_t day,
        TimeStamp *result);
+   static bool from_ymd(
+      std::string yearMonthDay,
+       TimeStamp *result);
 
    constexpr std::uint64_t nanos() const;
    constexpr std::uint64_t micros() const;

src/lib/md/MDConsumerInterop.cpp

@@ -128,6 +128,32 @@ CAPI bool md_consumer_is_heart_beating(const ColdHeader *header) {
    return is_heart_beat_ok(*header);
 }
 
+typedef void (*raw_receive_pb)(const char *, size_t);
+CAPI int md_consumer_read_trades_linux(
+    MDConsumer *consumer,
+    const char* symbol,
+    int venue_i,
+    int type_i,
+    int64_t &cursor,
+    raw_receive_pb cb) {
+   auto type = static_cast<SecurityType>(type_i);
+   auto venue = static_cast<Venue>(venue_i);
+   bts_assert(consumer);
+   google::protobuf::Arena arena;
+   co2::TradeList *ptrades =
+       google::protobuf::Arena::CreateMessage<co2::TradeList>(&arena);
+   auto channel =
+       consumer->get_channel(symbol, type, venue);
+   if (channel.is_initialized()) {
+      cursor = channel.read_trades(*ptrades->mutable_items(), cursor);
+      std::string buffer;
+      ptrades->SerializeToString(&buffer);
+      cb(buffer.c_str(), buffer.size());
+      return ptrades->items_size();
+   }
+   return 0;
+}
+
 typedef void (*raw_receive_pb)(const char *, size_t);
 CAPI int md_consumer_read_trades(
     MDConsumer *consumer,

src/lib/md/MarketDataGateway.cpp

@@ -1,6 +1,8 @@
+#include "lib/api/md/GlbxMarketDataConfig.pb.h"
 #if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
 #include "MarketDataGateway.h"
 #include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h"
+#include "lib/api/md/GlbxMarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfigUtils.h"
 #include "lib/api/md/MarketDataReplayConfig.pb.h"
@@ -40,6 +42,7 @@
 #include "lib/utils/thread_manager.h"
 #include "server/sparkd/md/platforms/emumba/EmumbaMain.h"
 #include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
+#include "lib/md/cme/CMEActivSession.h"
 
 #include <bits/stdint-uintn.h>
 #include <boost/filesystem.hpp>
@@ -135,15 +138,19 @@ void MarketDataGateway::run() {
    }
    LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
        os::get_thread_id(), os::get_thread_name());
+
    _start_cc(me_->md_config);
+   
    if (!me_->md_config.emumba_deribit().empty()) {
       bts_assert(me_->md_config.emumba_deribit().size() == 1);
       _start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
    }
+
    if (!me_->md_config.glbx().empty()) {
       bts_assert(me_->md_config.glbx().size() == 1);
-      _start_onix_glbx(me_->md_config.glbx().at(0));
+      _start_glbx(me_->md_config.glbx().at(0));
    }
+
    _start_replays(me_->md_config);
    while (true) {
       std::this_thread::sleep_for(std::chrono::seconds(1));
@@ -239,7 +246,26 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
    LOG(md, Info, "Unsubscribing listener {}", id);
    me_->listeners.erase(id);
 }
-/// glbx //////////////////////////////////////////////////////////////////////
+/// onix active //////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////
+static void run_active(bts::api::md::GlbxMarketDataConfig cfg, persistence::IDGenerator *sec_id_store) {
+   LOG(md, Info, "MarketDataGateway::run_active t_id:{} t_name", os::get_thread_id(),
+       os::get_thread_name());
+      bts::md::GeneralSecurityIndex sec_index;
+      bts::md::MDProducer md_producer;
+      std::shared_ptr<GeneralMarketDataListener> listener =std::make_shared<bts::md::GeneralMarketDataListener>(sec_index,md_producer,bts::api::VENUE_GLBX);
+      bts::md::cme::CMEActivSession::run(cfg.glbx_active_platform().host(), cfg.glbx_active_platform().user(), cfg.glbx_active_platform().password(), listener, sec_id_store);
+   // Timers
+//    auto &disp = utils::Dispatcher::current();
+//    utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
+//       for (auto &p : platforms)
+//          p->send_heartbeats();
+//    });
+//    heartbeat_timer.start();
+
+//    disp.run();
+}
+/// onix glbx //////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////////
 static void run_onixs() {
    LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
@@ -268,8 +294,29 @@ static void run_onixs() {
    disp.run();
 }
 
-void MarketDataGateway::_start_onix_glbx(
-    bts::api::md::GlbxMarketDataConfig cfg) {
+void MarketDataGateway::_start_active_glbx(bts::api::md::GlbxMarketDataConfig cfg){
+   LOG(md, Info,
+       "MarketDataGateway::_start_active_glbx starting t_id:{} t_name:{}. "
+       "cfg:\n{}",
+       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
+LOG(md, Info,
+       "MarketDataGateway::_start_active_glbx");
+   bts::md::run_active(cfg, &me_->sec_id_store);
+}
+void MarketDataGateway::_start_glbx( bts::api::md::GlbxMarketDataConfig cfg){
+   LOG(md, Info,
+       "MarketDataGateway::_start_glbx starting t_id:{} t_name:{}. "
+       "cfg:\n{}",
+       os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
+       if(cfg.has_glbx_active_platform()){
+            _start_active_glbx(cfg);
+       }
+       else if(cfg.has_glbx_onix_platform()){
+            _start_onix_glbx(cfg);
+       }
+
+}
+void MarketDataGateway::_start_onix_glbx( bts::api::md::GlbxMarketDataConfig cfg){
    LOG(md, Info,
        "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
        "cfg:\n{}",

src/lib/md/MarketDataGateway.h

@@ -35,6 +35,10 @@ class MarketDataGateway {
        bts::api::md::EmumbaDeribitMarketDataConfig cfg);
    static void _start_onix_glbx(
        bts::api::md::GlbxMarketDataConfig cfg);
+   void _start_glbx(
+       bts::api::md::GlbxMarketDataConfig cfg);
+   void _start_active_glbx(
+       bts::api::md::GlbxMarketDataConfig cfg);
    void _start_replays(const bts::api::md::MarketDataConfig &cfg);
 };
 

src/lib/md/cme/CMEActivSession.cpp

@@ -0,0 +1,91 @@
+#include "CMEActivSession.h"
+#include "lib/logging/logging.h"
+#include "lib/md/GeneralMarketDataListener.h"
+#include "lib/md/cme/CMEActivSessionHandler.h"
+#include "lib/md/cme/CMEActivSubscriptionHandler.h"
+#include "lib/md/cme/CMEActiveQueryHandler.h"
+#include "lib/md/cme/CMEActiveSnapshotHandler.h"
+#include <ActivMiddleware/ActivBase/Exception.h>
+#include <ActivMiddleware/ActivBase/FieldTypes/ActivFieldTypes.h>
+#include <ActivMiddleware/ActivBase/FieldTypes/FieldMap.h>
+#include <ActivOneApi/Constants/FieldIds.h>
+#include <ActivOneApi/Session.h>
+#include <exception>
+#include <ActivOneApi/Constants/DataSourceIds.h>
+
+namespace bts::md::cme {
+
+using Activ::FieldMap;
+using Activ::OneApi::Session;
+using Activ::OneApi::Handle;
+using std::string;
+void CMEActivSession::run(const string &host, const string &user, const string &password,std::shared_ptr<GeneralMarketDataListener> listner, persistence::IDGenerator *sec_id_store) {
+   try {
+      // Session callback handler.
+      CMEActivSessionHandler sessionHandler;
+
+      FieldMap fields;
+      // Construct the Session with a default configuration passing in our sessionHandler.
+      Session session(fields, sessionHandler);
+
+      FieldMap connectParameters;
+      
+      connectParameters.Set(Activ::OneApi::FID_HOST, host.c_str());
+      connectParameters.Set(Activ::OneApi::FID_USER_ID, user.c_str());
+      connectParameters.Set(Activ::OneApi::FID_PASSWORD, password.c_str());
+
+      // Connect synchronously with a 10 second timeout (will throw on failure).
+      session.Connect(connectParameters, 10000);
+      std::cerr << "connected" << std::endl;
+   
+      CMEActivQueryHandler query_handler(listner, sec_id_store);
+      // Specifying a data source and symbol.
+      // Activ::OneApi::Handle handle = session.Subscribe(Activ::OneApi::DATA_SOURCE_ACTIV,"CSIQ.Q",subscriptionHandler);
+      
+      // Activ::OneApi::Handle handle = session.Query(DATA_SOURCE_ACTIV,
+      //             "type=option and country=US and symbol=KC*",
+      //             subscriptionHandler);
+
+      md::cme::SnapshotHandler snapshotHandler;
+
+		// Request snapshot of the Microsoft US Equity topic.
+		// Returned Handle for a Snapshot is only required to cancel the request prior to a response.
+		// (void) session.Snapshot(DATA_SOURCE_ACTIV, "type=option and country=US and symbol=CSIQ/*", snapshotHandler);
+
+      // (void) session.Query(DATA_SOURCE_ACTIV, "symbol=CSIQ/*", query_handler);
+      (void) session.Query(DATA_SOURCE_ACTIV, "type=future and symbol=NK/*", query_handler);
+      // QueryResults queryResults;
+      // std::string query;
+      // while(true)
+      // {
+      //    query ="";
+      //    std::getline(std::cin, query);
+      //    if(query.substr(0,3) == "end")
+      //       break;
+      //    session.Query(DATA_SOURCE_ACTIV,
+      //                query,
+      //                queryResults);
+
+      //       // Prin  t out the topics.
+      //       for (const auto &message : queryResults)
+      //       {
+      //          std::cout << "Found symbol " << message.GetSymbol() << std::endl;
+      //       }
+      //    std::cout<<"END "<<std::endl;
+      // }
+      // Alternatively an asynchronous connection can be used.
+      // Connection completion is notified via ISessionHandler::OnSessionConnect().
+      // session.AsyncConnect(connectParameters);
+      session.Run();
+
+   } catch (const Activ::Exception &ex) {
+      std::cerr << "Error running CMEActivSession (activ): " << ex.GetMessage()
+                << ex.ToString()<< std::endl;
+   } catch (const std::exception &ex) {
+      std::cerr << "Error running CMEActivSession: " << ex.what() << std::endl;
+      // LOG(md, Error, "Error running CMEActivSession: {}",
+      //     std::string{ex.what()});
+   }
+}
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSession.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "lib/md/GeneralMarketDataListener.h"
+#include "lib/persistence/IDGenerator.h"
+#include <string>
+
+namespace bts::md::cme {
+
+class CMEActivSession {
+ public:
+   static void run(
+       const std::string &host,
+       const std::string &user,
+       const std::string &password,
+       std::shared_ptr<GeneralMarketDataListener> listner,
+      persistence::IDGenerator *sec_id_store);
+};
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSessionHandler.cpp

@@ -0,0 +1,33 @@
+#include "CMEActivSessionHandler.h"
+#include <ActivMiddleware/ActivBase/ActivBase.h>
+#include <ActivOneApi/Session.h>
+#include "CMEActivSubscriptionHandler.h"
+
+namespace bts::md::cme {
+
+using Activ::LogType;
+using Activ::StatusCode;
+using Activ::String;
+using Activ::OneApi::Session;
+
+void CMEActivSessionHandler::OnSessionLogMessage(
+    Session & /*session*/, const LogType log_type, const String &message) {
+   std::cout << Activ::LogTypeToString(log_type) << ": " << message
+             << std::endl;
+}
+
+void CMEActivSessionHandler::OnSessionConnect(Session & /*session*/) {
+   std::cerr << "connected" << std::endl;
+}
+
+void CMEActivSessionHandler::OnSessionDisconnect(Session & /*session*/) {
+   std::cout << "disconnected" << std::endl;
+}
+
+void CMEActivSessionHandler::OnSessionError(
+    Session & /*session*/, const StatusCode statusCode) {
+   std::string err{Activ::StatusCodeToString(statusCode)};
+   std::cerr << "CMEActivSessionHandler::OnSessionError. " << err << std::endl;
+}
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSessionHandler.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <ActivMiddleware/ActivBase/ActivBase.h>
+#include <ActivOneApi/ISessionHandler.h>
+
+namespace bts::md::cme {
+
+class CMEActivSessionHandler : public Activ::OneApi::ISessionHandler {
+ public:
+   static void run();
+
+   void OnSessionLogMessage(
+       Activ::OneApi::Session &session,
+       const Activ::LogType log_type,
+       const Activ::String &message) override;
+
+   void OnSessionConnect(Activ::OneApi::Session &session) override;
+   void OnSessionDisconnect(Activ::OneApi::Session &session) override;
+   void OnSessionError(
+       Activ::OneApi::Session &session,
+       const Activ::StatusCode statusCode) override;
+};
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActivSubscriptionHandler.h

@@ -0,0 +1,365 @@
+#pragma once
+#include "lib/core/TimeStamp.h"
+#include "lib/domain/AggressorSide.h"
+#include "lib/domain/Book.h"
+#include "lib/domain/BookLevel.h"
+#include "lib/domain/FutureSecurity.h"
+#include "lib/domain/Quantity.h"
+#include "lib/domain/SecurityID.h"
+#include "lib/domain/Side.h"
+#include "lib/domain/TradeRecord.h"
+#include "lib/md/GeneralMarketDataListener.h"
+#include "lib/md/MarketDataGateway.h"
+#include "lib/orderman/orderman_utils/Book.h"
+#include "lib/persistence/IDGenerator.h"
+#include "lib/utils/bts_assert.h"
+#include <ActivOneApi/MarketData/EventTypes.h>
+#include <ActivOneApi/MarketData/FieldIds.h>
+#include <ActivOneApi/MarketData/Separators.h>
+#include <cassert>
+#include <iostream>
+#include <string>
+#define UNUSED(x) (void)(x)
+#include "ActivOneApi/MarketData/MarketData.h"
+#include <ActivMiddleware/ActivBase/ActivBase.h>
+
+#include "ActivOneApi/OneApi.h"
+#include <ActivOneApi/ISessionHandler.h>
+#include <ActivOneApi/ISubscriptionHandler.h>
+#include <ActivOneApi/Messages/RefreshMessage.h>
+#include <ActivOneApi/Messages/UpdateMessage.h>
+using namespace Activ;
+using namespace Activ::OneApi;
+namespace bts::md::cme {
+class CMEActivSubscriptionHandler : public Activ::OneApi::ISubscriptionHandler {
+
+   FieldListValidator m_fieldListValidator;
+
+   Activ::OneApi::TopicSubscriptionState *m_state;
+
+ public:
+   std::shared_ptr<GeneralMarketDataListener> listner;
+   persistence::IDGenerator *sec_id_store;
+
+   void init(
+       std::shared_ptr<GeneralMarketDataListener> listner,
+       persistence::IDGenerator *id_store,
+       Activ::OneApi::TopicSubscriptionState *m_state) {
+      this->listner = listner;
+      sec_id_store = id_store;
+      this->m_state = m_state;
+   }
+   CMEActivSubscriptionHandler() {}
+   CMEActivSubscriptionHandler(
+       std::shared_ptr<GeneralMarketDataListener> listner,
+       persistence::IDGenerator *sec_id_store,
+       Activ::OneApi::TopicSubscriptionState *m_state)
+       : m_state(m_state), listner(listner), sec_id_store(sec_id_store) {}
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+   std::string removeCharacters(const std::string &input) {
+      std::string result;
+
+      for (char c : input) {
+         if (c != '/' && c != '.') {
+            result += c;
+         } else if (c == '.') {
+            break; // Stop adding characters after encountering a dot
+         }
+      }
+
+      return result;
+   }
+   void OnSubscriptionRefresh(
+       const RefreshMessage &refreshMessage, const Context &context) {
+      UNUSED(context);
+      std::cout << "REFRESH received for " << refreshMessage.m_symbol
+                << std::endl;
+      std::cout << "Symbol .................... " << refreshMessage.m_symbol
+                << std::endl;
+      std::cout << "Topic subscription state .. "
+                << TopicSubscriptionStateToString(
+                       refreshMessage.m_topicSubscriptionState)
+                << std::endl;
+      StatusCode statusCode = m_fieldListValidator.Initialize(refreshMessage);
+
+      for (const Activ::OneApi::FlvDetail::Field &field :
+           m_fieldListValidator) {
+            if(field.IsDefined())
+         std::cout<<"onsub Feild: "<<MarketData::GetFieldName(field.GetId())<<", Value: "<<field.GetValue<std::string>() <<std::endl;
+      }
+
+      if (STATUS_CODE_SUCCESS != statusCode) {
+         std::cout << "ERROR decoding field data for "
+                   << refreshMessage.m_symbol << " - "
+                   << StatusCodeToString(statusCode) << std::endl;
+         return;
+      }
+      const Activ::OneApi::FlvDetail::Field *pField;
+      statusCode = m_fieldListValidator.GetField(
+          MarketData::FID_EXPIRATION_DATE, pField);
+
+      if (STATUS_CODE_SUCCESS != statusCode) {
+         std::cout << "ERROR fetching "
+                   << "exp date"
+                   << " field for " << refreshMessage.m_symbol << " - "
+                   << StatusCodeToString(statusCode) << std::endl;
+         return;
+      }
+      std::string exp_date = pField->GetValue<String>();
+      core::TimeStamp exp_time;
+      if (!core::TimeStamp::from_ymd(exp_date, &exp_time)) {
+         return;
+      }
+
+      String result = removeCharacters(refreshMessage.m_symbol);
+      domain::SecurityID sec_id{
+          sec_id_store->get_or_add(api::domain::VENUE_GLBX, result)};
+      std::vector<domain::Security> complete_security_list;
+      domain::Security &sec = complete_security_list.emplace_back();
+      sec.security_common.venue = api::domain::VENUE_GLBX;
+      sec.security_common.id = sec_id;
+      sec.security_common.exchange_sec_id =
+          refreshMessage.ToString(); // Need to fix this
+      sec.security_common.desc = result;
+      sec.security_common.symbol = result;
+      sec.security_common.asset = "a1";
+      sec.security_common.group = result;
+
+      sec.security_common.min_price_increment = 0.1;
+      sec.security_common.min_quantity = 1;
+      sec.security_common.min_quantity_increment = 0.1;
+      sec.security_common.currency = "name";
+      auto &t = sec.security.emplace<domain::FutureSecurity>();
+      t.expiration = exp_time;
+      std::vector<const domain::Security *> const_security_list;
+      for (const auto &security : complete_security_list) {
+         const_security_list.push_back(&security);
+      }
+      listner->on_securities(const_security_list);
+
+      
+      // Alternatively use random access.
+
+      // Print out the current value for FID_TRADE. FID_TRADE is defined in the Activ::MarketData namespace.
+      // ProcessField(
+      //     MarketData::FID_EXPIRATION_DATE, "Exp ", refreshMessage.m_symbol);
+   }
+   template <typename T>
+   bool _get_field(String field_name, Activ::OneApi::FieldId fieldId, T &res ){
+      const Activ::OneApi::FlvDetail::Field *pField;
+      auto statusCode =
+          m_fieldListValidator.GetField(fieldId, pField);
+      if (STATUS_CODE_SUCCESS != statusCode) {
+         std::cout << "ERROR fetching "
+                   << field_name
+                   << " field Error: "
+                   << StatusCodeToString(statusCode) << std::endl;
+         return false;
+      }
+      res = pField->GetValue<T>();
+      return true;
+   }
+   bool _parse_trade(std::vector<const domain::TradeRecord *> &trades, domain::SecurityID id) {
+      UNUSED(trades);
+      for (const Activ::OneApi::FlvDetail::Field &field :
+           m_fieldListValidator) {
+         if (field.IsDefined()) {
+
+            std::cout << "Has Feild: "
+                      << MarketData::GetFieldName(field.GetId())
+                      << ", Value: " << field.GetValue<std::string>()
+                      << std::endl;
+         }
+      }
+      std::vector<domain::TradeRecord> parsed_trades;
+      auto &t = parsed_trades.emplace_back();
+      double tp;
+      if(!_get_field<double>("FID_TRADE", MarketData::FID_TRADE, tp))
+         return false;
+      uint ts;
+      if(!_get_field<uint>("FID_TRADE_SIZE", MarketData::FID_TRADE_SIZE, ts))
+         return false;
+      String time, date;
+      if(!_get_field<String>("FID_TRADE_TIME", MarketData::FID_TRADE_TIME, time))
+         return false;
+      if(!_get_field<String>("FID_TRADE_DATE", MarketData::FID_TRADE_DATE, date))
+         return false;
+      
+      t.trade_price=bts::domain::Price::from_double(tp);
+      t.trade_quantity=bts::domain::Quantity(ts);
+      t.sec_id=id;
+      t.agg_side=bts::domain::AggressorSide::Buy;
+      t.time_stamp=core::TimeStamp::from_iso_string(date+"T"+time);;
+      trades.push_back(&parsed_trades[0]);
+      bts_assert(trades[0]->sec_id==id);
+      listner->on_trades(trades);
+      return true;
+   }
+   // void _parse_side()
+   bool _parse_book(domain::Book &result, size_t n_bid_levels = 5) {
+      UNUSED(n_bid_levels);
+      uint ask_size, bid_size;
+      if(!_get_field<uint>("FID_ASK_SIZE", MarketData::FID_ASK_SIZE, ask_size))
+         return false;
+      if(!_get_field<uint>("FID_BID_SIZE", MarketData::FID_BID_SIZE, bid_size))
+         return false;
+      double bid_val,ask_val;
+      if(!_get_field<double>("FID_BID", MarketData::FID_BID, bid_val))
+         return false;
+      if(!_get_field<double>("FID_ASK", MarketData::FID_ASK, ask_val))
+         return false;
+      domain::BookSideT<10> &b = result.bids();
+      domain::BookSideT<10> &a = result.asks();
+      domain::BookLevel b_l;
+      b_l.price=domain::Price::from_double(bid_val);
+      b_l.total_quantity=domain::Quantity(bid_size);
+      domain::BookLevel a_l;
+      a_l.price=domain::Price::from_double(ask_val);
+      a_l.total_quantity=domain::Quantity(ask_size);
+      b[0]=b_l;
+      a[0]=a_l;
+
+      return true;
+   }
+   // ---------------------------------------------------------------------------------------------------------------------------------
+   void ProcessField(
+       const FieldId fieldId, const String &fieldName, const String &symbol) {
+      const FieldListValidator::Field *pField;
+
+      // FieldListValidator::GetField() provides access to the internally stored cached field.
+      StatusCode statusCode = m_fieldListValidator.GetField(fieldId, pField);
+      if (STATUS_CODE_SUCCESS != statusCode) {
+         std::cout << "ERROR fetching " << fieldName << " field for " << symbol
+                   << " - " << StatusCodeToString(statusCode) << std::endl;
+         return;
+      }
+
+      std::cout << fmt::format("{:.<27s} ", fieldName + " ");
+
+      if (!pField->IsDefined()) {
+         std::cout << "undefined" << std::endl;
+      } else {
+         // Field contents are only valid if the Field is defined.
+         std::cout << pField->GetValue<String>() << std::endl;
+      }
+   }
+   void OnSubscriptionUpdate(
+       const UpdateMessage &updateMessage, const Context &context) {
+      UNUSED(context);
+      m_fieldListValidator.ApplyUpdate(updateMessage);
+      String result = removeCharacters(updateMessage.m_symbol);
+      auto id = sec_id_store->get_id(api::domain::VENUE_GLBX, result);
+      assert(id.has_value());
+
+      for (const Activ::OneApi::FlvDetail::Field &field :
+           m_fieldListValidator) {
+            if(field.IsDefined())
+         std::cout<<"Update Feild: "<<MarketData::GetFieldName(field.GetId())<<", Value: "<<field.GetValue<std::string>() <<std::endl;
+      }
+      switch (updateMessage.m_eventType) {
+      case MarketData::EVENT_TYPE_BBO_QUOTE:
+      case MarketData::EVENT_TYPE_BBO_DEPTH: {
+         domain::Book book;
+         book.set_security_id(domain::SecurityID(id.value()));
+         if (_parse_book(book)) {
+            listner->on_book(book);
+            std::cout << "Book UPDATE received for " << updateMessage.m_symbol
+                      << std::endl;
+         }
+         break;
+      }
+      case MarketData::EVENT_TYPE_TRADE:
+      {
+         std::vector<const domain::TradeRecord *> trades;
+         if(_parse_trade(trades,domain::SecurityID(id.value()))){
+            
+            std::cout << "trade UPDATE received for " << updateMessage.m_symbol
+                      << std::endl;
+         }
+         break;
+      }
+      default:
+         std::cout << "case not defined \n";
+      }
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnSubscriptionTopicStatus(
+       const TopicStatusMessage &topicStatusMessage, const Context &context) {
+      UNUSED(context);
+      std::cout << "TOPIC STATUS received for " << topicStatusMessage.m_symbol
+                << " "
+                << TopicSubscriptionStateToString(
+                       topicStatusMessage.m_topicSubscriptionState)
+                << std::endl;
+
+      *m_state = topicStatusMessage.m_topicSubscriptionState;
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnSubscriptionError(
+       const StatusCode statusCode, const Context &context) {
+      UNUSED(context);
+      std::cout << "Subscription error " << StatusCodeToString(statusCode)
+                << std::endl;
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnSubscriptionFailure(
+       const StatusCode statusCode, const Context &context) {
+      UNUSED(context);
+      std::cout << "Subscription failure " << StatusCodeToString(statusCode)
+                << std::endl;
+   }
+   // void OnSubscriptionUpdate(
+   //     const Activ::OneApi::UpdateMessage &updateMessage,
+   //     const Activ::OneApi::Context &context) {
+   //    UNUSED(context);
+   //    std::cout << "UPDATE: " << updateMessage.m_symbol
+   //              << updateMessage.ToString() << std::endl;
+
+   //    // ApplyUpdate is used here, unlike Initialize, ApplyUpdate does not clear existing
+   //    // field content.
+   //    // m_fieldListValidator.ApplyUpdate(updateMessage);
+
+   //    // ... use m_fieldListValidator to access content.
+   // }
+   // void OnSubscriptionRefresh(
+   //     const Activ::OneApi::RefreshMessage &refreshMessage,
+   //     const Activ::OneApi::Context &context) {
+   //    UNUSED(context);
+   //    std::cout << "REFRESH: " << refreshMessage.m_symbol << std::endl;
+
+   //    // Access to field data is obtained using a FieldListValidator.
+   //    // m_fieldListValidator.Initialize(refreshMessage);
+
+   //    // ... use m_fieldListValidator to access content.
+   // }
+
+   // void OnSubscriptionTopicStatus(
+   //     const TopicStatusMessage &topicStatusMessage, const Context &context) {
+   //    UNUSED(context);
+   //    // If topicStatusMessage.m_topicSubscriptionState is not TOPIC_SUBSCRIPTION_STATE_OK
+   //    // then any cached fields may be stale.
+   //    std::cout << "TOPIC STATUS: " << topicStatusMessage.m_symbol
+   //              << ", state=" << topicStatusMessage.ToString() << std::endl;
+   // }
+   // void OnSubscriptionError(
+   //     const StatusCode statusCode, const Context &context) {
+   //    std::cout << "error :" << StatusCodeToString(statusCode) << std::endl;
+   //    UNUSED(statusCode);
+   //    UNUSED(context);
+   // }
+   // void OnSubscriptionFailure(
+   //     const StatusCode statusCode, const Context &context) {
+   //    std::cout << "Failure :" << StatusCodeToString(statusCode) << std::endl;
+   //    UNUSED(statusCode);
+   //    UNUSED(context);
+   // }
+};
+
+} // namespace bts::md::cme

src/lib/md/cme/CMEActiveQueryHandler.h

@@ -0,0 +1,108 @@
+#pragma once
+#include "lib/md/cme/CMEActivSubscriptionHandler.h"
+#include "lib/md/GeneralMarketDataListener.h"
+#include <ActivMiddleware/CxxYZ/unordered_map.h>
+#define UNUSED(x) (void)(x)
+#include "ActivOneApi/OneApi.h"
+namespace bts::md::cme {
+class CMEActivQueryHandler : public Activ::OneApi::ITopicQueryHandler {
+   class TopicDetails {
+    public:
+      //! The subscription handle for this topic
+      Activ::OneApi::ScopedHandle m_handle;
+      CMEActivSubscriptionHandler sub_handler;
+      //! The current topic subscription state. Is this topic waiting for a refresh.
+      Activ::OneApi::TopicSubscriptionState m_state;
+
+      /**
+		 *	@brief	Has this topic been seen for the current topic query. Whilst a query is in an error state there may be missed
+		 *			topic deletions, these need to be cleaned up once the OnTopicQueryComplete callback is received.
+		 */
+      size_t m_queryVersion;
+   };
+   //! Used to track how many times the query has restarted.
+   // size_t m_queryVersion;
+   //! Map of subscriptions.
+   typedef Activ::CxxYZ::unordered_map<Activ::String, TopicDetails> SubscriptionMap;
+   SubscriptionMap m_subscriptionMap;
+   //! Used to track how many times the query has restarted.
+   size_t m_queryVersion;
+   public:
+   std::shared_ptr<GeneralMarketDataListener> listner;
+   persistence::IDGenerator *sec_id_store;
+   CMEActivQueryHandler(
+       std::shared_ptr<GeneralMarketDataListener> listner,
+       persistence::IDGenerator *sec_id_store)
+       : m_queryVersion(0), listner(listner), sec_id_store(sec_id_store) {}
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnTopicQueryStart(const Context &context) {
+      UNUSED(context);
+      std::cout << "QUERY start" << std::endl;
+      ++m_queryVersion;
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnTopicQueryAdd(Topic &topic, const Context &context) {
+      const String &symbol = topic.GetSymbol();
+      TopicDetails &topicDetails = m_subscriptionMap[symbol];
+
+      if (!topicDetails.m_handle) {
+         topicDetails.m_queryVersion = m_queryVersion;
+         topicDetails.m_state = TOPIC_SUBSCRIPTION_STATE_PENDING;
+         topicDetails.sub_handler = CMEActivSubscriptionHandler(listner, sec_id_store, &topicDetails.m_state);
+         topicDetails.m_handle =
+             context.GetSession().Subscribe(DATA_SOURCE_ACTIV, symbol, topicDetails.sub_handler);
+      } else {
+         topicDetails.m_queryVersion = m_queryVersion;
+      }
+   }
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnTopicQueryRemove(const TopicId &topicId, const Context &context) {
+      UNUSED(context);
+      std::cout << "QUERY topic removed: " << topicId.m_symbol << std::endl;
+
+      SubscriptionMap::iterator iter = m_subscriptionMap.find(topicId.m_symbol);
+      if (m_subscriptionMap.end() != iter)
+         m_subscriptionMap.erase(iter);
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnTopicQueryComplete(const Context &context) {
+      std::cout << "QUERY complete" << std::endl;
+      UNUSED(context);
+      // Close the subscriptions for any topics that haven't been received.
+      SubscriptionMap::iterator iter = m_subscriptionMap.begin();
+      while (m_subscriptionMap.end() != iter) {
+         if (iter->second.m_queryVersion != m_queryVersion)
+            m_subscriptionMap.erase(iter++);
+         else
+            ++iter;
+      }
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnTopicQueryError(const StatusCode statusCode, const Context &context) {
+      std::cout << "QUERY error " << StatusCodeToString(statusCode)
+                << std::endl;
+      UNUSED(context);
+   }
+
+   // ---------------------------------------------------------------------------------------------------------------------------------
+
+   void OnTopicQueryFailure(
+       const StatusCode statusCode, const Context &context) {
+      std::cout << "QUERY failure " << StatusCodeToString(statusCode)
+                << std::endl;
+
+      // Shutdown session.
+      Session::Shutdown(context.GetSession());
+      UNUSED(context);
+   }
+};
+} // namespace bts::md::cme

src/lib/md/cme/CMEActiveSnapshotHandler.h

@@ -0,0 +1,100 @@
+/**
+ *	@file	source/ActivOnePlatform/Samples/SnapshotFieldListValidatorSample/SnapshotHandler.h
+ *
+ *	@brief	SnapshotHandler definition.
+ */
+
+#if (!defined ACTIV_SAMPLE_SNAPSHOT_HANDLER_H)
+#define ACTIV_SAMPLE_SNAPSHOT_HANDLER_H
+
+#include "ActivOneApi/OneApi.h"
+#include "ActivOneApi/MarketData/MarketData.h"
+
+
+#include <ostream>
+
+using namespace Activ;
+using namespace Activ::OneApi;
+namespace bts::md::cme {
+//! SnapshotHandler.
+class SnapshotHandler : public ISnapshotHandler
+{
+public:
+	virtual void OnSnapshot(const SnapshotMessage &snapshotMessage, const Context &context) ACTIV_OVERRIDE;
+	virtual void OnSnapshotFailure(const SnapshotFailureMessage &snapshotFailureMessage, const Context &context) ACTIV_OVERRIDE;
+
+private:
+	void ProcessField(const FieldId fieldId, const String &fieldName, const String &symbol);
+
+	// FieldListValidator will parse SnapshotMessage, if possible reuse FieldListValidator objects.
+	FieldListValidator	m_fieldListValidator;
+};
+
+// ---------------------------------------------------------------------------------------------------------------------------------
+
+inline void SnapshotHandler::OnSnapshot(const SnapshotMessage &snapshotMessage, const Context &context)
+{
+	std::cout << "SNAPSHOT received for " << snapshotMessage.m_symbol << std::endl;
+	std::cout << "Symbol .................... " << snapshotMessage.m_symbol << std::endl;
+
+	// Initialize will clear any existing cache fields. SnapshotMessage contains a complete topic field list if no filters have
+	// been requested.
+	StatusCode statusCode = m_fieldListValidator.Initialize(snapshotMessage);
+
+	if (STATUS_CODE_SUCCESS != statusCode)
+	{
+		std::cout << "ERROR decoding field data for " << snapshotMessage.m_symbol << " - " << StatusCodeToString(statusCode) << std::endl;
+		return;
+	}
+     for (const Activ::OneApi::FlvDetail::Field &field :
+           m_fieldListValidator) {
+            if(field.IsDefined())
+         std::cout<<"Has snapshot Feild: "<<MarketData::GetFieldName(field.GetId())<<", Value: "<<field.GetValue<std::string>() <<std::endl;
+      }
+
+	// Print out the current value for FID_TRADE. FID_TRADE is defined in the Activ::MarketData namespace.
+	ProcessField(MarketData::FID_TRADE, "FID_TRADE", snapshotMessage.m_symbol);
+
+	std::cout << std::endl;
+
+	context.GetSession().CloseHandle(context.GetHandle());
+}
+
+// ---------------------------------------------------------------------------------------------------------------------------------
+
+inline void SnapshotHandler::OnSnapshotFailure(const SnapshotFailureMessage &snapshotFailureMessage, const Context &context)
+{
+	std::cout << "SNAPSHOT FAILURE for " << snapshotFailureMessage.m_symbol << std::endl;
+	std::cout << snapshotFailureMessage.ToString() << std::endl;
+
+	context.GetSession().CloseHandle(context.GetHandle());
+}
+
+// ---------------------------------------------------------------------------------------------------------------------------------
+
+inline void SnapshotHandler::ProcessField(const FieldId fieldId, const String &fieldName, const String &symbol)
+{
+	const FieldListValidator::Field *pField;
+
+	// FieldListValidator::GetField() provides access to the internally stored cached field.
+	StatusCode statusCode = m_fieldListValidator.GetField(fieldId, pField);
+	if (STATUS_CODE_SUCCESS != statusCode)
+	{
+		std::cout << "ERROR fetching " << fieldName << " field for " << symbol << " - " << StatusCodeToString(statusCode) << std::endl;
+		return;
+	}
+
+
+	if (!pField->IsDefined())
+	{
+		std::cout << "undefined" << std::endl;
+	}
+	else
+	{
+		// Field contents are only valid if the Field is defined.
+		std::cout << pField->GetValue<String>() << std::endl;
+	}
+}
+
+}
+#endif // (!defined ACTIV_SAMPLE_SNAPSHOT_HANDLER_H)

src/lib/md/recording/MDRecorder.cpp

@@ -1,4 +1,4 @@
-#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
+#if !defined(_WIN32) //&& defined(SPARK_ENABLE_CCAPI)
 #include "MDRecorder.h"
 #include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfigUtils.h"
@@ -184,7 +184,7 @@ void MDRecorder::_poll_md() {
       uint64_t now_ns = core::TimeStamp::now_ns();
       for (auto &kv : me->handles) {
          MDRecordingSecEntry &md_entry = kv.second;
-         if(me->settings.n_book_levels > 0)
+         if (me->settings.n_book_levels > 0)
             _poll_books(md_entry);
          _poll_trades(md_entry);
          _poll_misc(md_entry);

src/lib/md/recording/MDReplayer.cpp

@@ -1,4 +1,4 @@
-#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
+#if !defined(_WIN32) //&& defined(SPARK_ENABLE_CCAPI)
 #include "MDReplayer.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/system_config.pb.h"

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -381,6 +381,9 @@ void OrderManager::on_gateway_state(
     const mercury::Update<GatewayStatusChangeEvent> &m) {
    if (m.events)
       for (auto const &ev : *m.events) {
+         LOG(OrderMan, Info,
+             " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
+             ev.is_connected);
          auto &sr = me->services_data.get_session(ev.gw_name.value());
          sr.gw_connected = ev.is_connected;
       }

src/lib/orderman/ccapi/CCAPIDeribitOrderMux.cpp

@@ -1,3 +1,5 @@
+#ifdef SPARK_ENABLE_CCAPI
+
 #include "CCAPIDeribitOrderMux.h"
 #include "CCAPIOrderState.h"
 #include "CCAPISingleOrderManager.h"
@@ -953,4 +955,6 @@ void CCAPIDeribitOrderMux::_send_buffered_orders(uint32_t sec_id) {
        "n_buffered_orders: {}",
        me->buffered_creates.size());
 }
-} // namespace bts::orderman
+} // namespace bts::orderman
+
+#endif

src/lib/orderman/ccapi/CCAPIDeribitOrderMux.h

@@ -1,4 +1,6 @@
 #pragma once
+#ifdef SPARK_ENABLE_CCAPI
+
 #include "ICCAPIOrderMux.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/CreateOrderMsg.h"
@@ -72,4 +74,6 @@ class CCAPIDeribitOrderMux : public ICCAPIOrderMux {
    void _handle_generic_pvt_request(const nlohmann::json &json_obj);
 };
 
-} // namespace bts::orderman
+} // namespace bts::orderman
+
+#endif

src/lib/orderman/events/GatewayStatusChangeEvent.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "lib/logging/logging.h"
 #include "lib/mercury/synopsis/CacheLast.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include <memory>
@@ -14,7 +15,11 @@ class GatewayStatusChangeEvent
    using SynopsisPolicy = mercury::CacheLast;
 
    GatewayStatusChangeEvent(GatewayName const &n, bool c)
-       : gw_name(n), is_connected(c) {}
+       : gw_name(n), is_connected(c) {
+      LOG(OrderMan, Info,
+          "GatewayStatusChangeEvent::cons gw_name: {} is_connected: {}",
+          gw_name, is_connected);
+   }
 
    GatewayName gw_name;
    bool is_connected;

src/lib/orderman/gateways/CCAPIExecutionGateway.cpp

@@ -1,3 +1,5 @@
+#ifdef SPARK_ENABLE_CCAPI
+
 #include "CCAPIExecutionGateway.h"
 #include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/api/services/AdminService.pb.h"
@@ -492,4 +494,6 @@ SingleOrderMux *CCAPIExecutionGateway::_find_mux_from_orderkey(OrderKey key) {
    return _find_mux(it->second.routing_record);
 }
 
-} // namespace bts::orderman
+} // namespace bts::orderman
+
+#endif

src/lib/orderman/gateways/CCAPIExecutionGateway.h

@@ -1,4 +1,5 @@
 #pragma once
+#ifdef SPARK_ENABLE_CCAPI
 
 #include "IExecutionGateway.h"
 #include "lib/api/services/AdminService.pb.h"
@@ -73,3 +74,5 @@ class CCAPIExecutionGateway : public IExecutionGateway {
 
 } // namespace orderman
 } // namespace bts
+
+#endif

src/lib/orderman/gateways/EMExecutionGateway.cpp

@@ -1,3 +1,4 @@
+#ifdef SPARK_ENABLE_CCAPI
 #include "EMExecutionGateway.h"
 #include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/api/services/AdminService.pb.h"
@@ -705,4 +706,6 @@ const api::Security *EMExecutionGateway::_get_security(
 }
 
 } // namespace orderman
-} // namespace bts
+} // namespace bts
+
+#endif

src/lib/orderman/gateways/ExecutionGatewayFactory.cpp

@@ -34,9 +34,11 @@ std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
          return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(
              conn, pl, o);
       return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
-   } else if (conn.venue() == bts::api::VENUE_DERIBIT)
-      //   return std::make_unique<EMExecutionGateway>(o);
+   } else if (conn.venue() == bts::api::VENUE_DERIBIT) {
+#ifdef SPARK_ENABLE_CCAPI
       return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
+#endif
+   }
    return std::make_unique<FakeExecutionGateway>(o);
 }
 

src/lib/orderman/gateways/GLBXiLink3ExecutionGatewayState.h

@@ -1,5 +1,4 @@
 #pragma once
-#ifdef SPARK_ENABLE_CCAPI
 
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
@@ -26,7 +25,6 @@ ENUM_CLASS_DEF(
 #undef FOR_EACH_GLBX_ILINK3_EXECUTION_GATEWAY_STATE
 
 } // namespace bts::orderman::ilink3
-#endif
 
 FMT_IMPL(
     bts::orderman::ilink3::GLBXiLink3ExecutionGatewayState, s, "{}", s.name());

src/lib/orderman/gateways/sim/SimExecutionGateway.cpp

@@ -30,6 +30,7 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/utils/thread_manager.h"
 #include "server/sparkd/md/platforms/emumba/IEMOrderMuxer.h"
 
 #include <bits/stdint-uintn.h>
@@ -347,8 +348,10 @@ void SimExecutionGateway::on_admin_data(
        bts::os::get_thread_name());
 }
 
-void SimExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {
-   bts_assert(false);
+void SimExecutionGateway::send_fix(const jetfix::OutFIXPtr &p, trace::CorID) {
+   LOG(OrderMan, Warn, "SimExecutionGateway::send_fix p sz: {}", p->fix.size());
+   // TODO
+   //bts_assert(false);
 }
 
 // This is where the messages are sent to the exchange. We simply write the
@@ -364,7 +367,7 @@ void SimExecutionGateway::send_msgs(
        "SimExecutionGateway::send_msgs tid: {} tname: {} count: {}",
        bts::os::get_thread_id(), bts::os::get_thread_name(), msg_vec.size());
    for (auto &m : msg_vec) {
-      LOG(OrderMan, Diag,
+      LOG(OrderMan, Info,
           "SimExecutionGateway::send_msgs tid: {} tname: {} m: {}",
           bts::os::get_thread_id(), bts::os::get_thread_name(), m);
       me->buffered_order_ops.push(m);
@@ -373,6 +376,10 @@ void SimExecutionGateway::send_msgs(
 
 OnExeReport &SimExecutionGateway::on_exe_report() { return me->on_exe_report; }
 OnGwStatusChange &SimExecutionGateway::on_gw_status_change() {
+   std::string tid_name = "SimExecutionGateway::_send_gw_status_once";
+   bts::threading::make_thread(tid_name, [&]() {
+      _send_gw_status_once();
+   }).detach();
    return me->on_gw_status_change;
 }
 OnCancelReject &SimExecutionGateway::on_cancel_reject() {
@@ -675,6 +682,13 @@ SimExecutionGateway::_find_order(OrderKey key) const {
        nullptr, nullptr);
 }
 
+void SimExecutionGateway::_send_gw_status_once() {
+   auto sleep_period = std::chrono::milliseconds(2500);
+   std::this_thread::sleep_for(sleep_period);
+   // fire the status and exit the thread
+   me->on_gw_status_change(true);
+}
+
 // this returns the security key from an OrdMsg. The reason it has more code
 // than needed is because CancelOrderMsg might not have the security set.
 // therefore we need to find the security from the order key

src/lib/orderman/gateways/sim/SimExecutionGateway.h

@@ -76,6 +76,7 @@ class SimExecutionGateway : public IExecutionGateway {
    std::tuple<SEGSecurityData const *, SEGOrderState const *> _find_order(
        OrderKey key) const;
    std::optional<SecurityKey> _extract_sec_key(const OrderMsg &msg);
+   void _send_gw_status_once();
 
  private:
    struct PImpl;

src/lib/orderman/gateways/sim/SimExecutionGatewayFactory.cpp

@@ -61,6 +61,8 @@ SimExecutionGatewayFactory *SimExecutionGatewayFactory::get() {
 }
 
 bool SimExecutionGatewayFactory::is_sim_execution_on(api::Venue venue) {
+   // TODO: Arham, fix this !
+   return true;
    const auto &sim_exec = config::get<bts::config::SimExecution>();
    for (const auto &e : sim_exec.entries()) {
       if (e.venue() == venue) {

src/lib/orderman/ilink3/IL3State.h

@@ -1,5 +1,4 @@
 #pragma once
-#ifdef SPARK_ENABLE_CCAPI
 
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
@@ -27,6 +26,5 @@ ENUM_CLASS_DEF(IL3State, FOR_EACH_IL3_STATE)
 #undef FOR_EACH_IL3_STATE
 
 } // namespace bts::orderman::ilink3
-#endif
 
 FMT_IMPL(bts::orderman::ilink3::IL3State, s, "{}", s.name());

src/lib/orderman/orders/GLBXNativeOrder.cpp

@@ -4,11 +4,14 @@
 #include "lib/config/protos/orderman_cfg.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
 #include "lib/jetfix/codec/FIXFragment.h"
 #include "lib/jetfix/codec/FieldDefinitions.h"
 #include "lib/logging/logging.h"
 #include "lib/orderman/OrderRouter.h"
 #include "lib/orderman/OrderServices.h"
+#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
 #include "lib/orderman/order_factory/IOrderFactory.h"
 #include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include "lib/orderman/orderman_utils/fix_order_codec.h"
@@ -65,6 +68,8 @@ struct GLBXNativeOrder::PImpl {
    std::optional<api::orderman::Update> buffered_update;
    RoutingRecord routing_record;
    bool stop_order_still_has_stop_px = true;
+   const bool in_sim_mode =
+       SimExecutionGatewayFactory::is_sim_execution_on(api::VENUE_GLBX);
 };
 
 GLBXNativeOrder::GLBXNativeOrder() : me(new PImpl()) {}
@@ -471,8 +476,13 @@ OrderOpResult GLBXNativeOrder::on_activate(
              "delta:{}",
              string{corid.channel}, corid.sn, corid.app_arrival.to_uint64(),
              now.to_uint64(), (now - corid.app_arrival).nanoseconds());
-
+         // in sim mode, we dont need this,
          me->encoder->send_create(*me->services, *me->session_services);
+         if (me->in_sim_mode) {
+            CreateOrderMsg c_msg = {};
+            _create_create_msg(c_msg);
+            me->services->queue_order_msg(*me->session_services, c_msg);
+         }
          me->services->governor(*me->session_services).record_sent();
          me->state.set_status(ORDER_STATUS_PENDING_WORKING);
 
@@ -630,6 +640,16 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
                me->services->report_definition(me->definition);
                me->encoder->update_cancel_replace_stop_px(
                    stop_px, me->price_encoding_precision);
+               if (me->in_sim_mode) {
+                  CancelReplaceOrderMsg cxl_repl_msg = {};
+                  _create_create_msg(cxl_repl_msg.create_fields);
+                  Price new_p = Price(update.price());
+                  Quantity new_q = bts::orderman::get_update_quantity(update);
+                  cxl_repl_msg.create_fields.price = new_p;
+                  cxl_repl_msg.create_fields.quantity = new_q;
+                  me->services->queue_order_msg(
+                      *me->session_services, cxl_repl_msg);
+               }
             }
          } else if (
              update.custom().custom_case() ==
@@ -773,6 +793,13 @@ OrderOpResult GLBXNativeOrder::on_cancel() {
    case ORDER_STATUS_WORKING: {
       me->encoder->update_cancel_encoding(*me->services);
       me->encoder->send_cancel(*me->services, *me->session_services);
+      // TODO: check for sim mode here
+      if (me->in_sim_mode) {
+         CancelOrderMsg cxl_msg = {};
+         cxl_msg.key = key();
+         LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl");
+         me->services->queue_order_msg(*me->session_services, cxl_msg);
+      }
       break;
    }
 
@@ -1326,5 +1353,37 @@ void GLBXNativeOrder::fixup_routing_record(api::orderman::RoutingRecord *rr) {
                                           .fallback_cust_order_handling_inst());
 }
 
+// TODO, make it static
+NativeOrderType get_glbx_native_ord_type(const OrderDefinition &def) {
+   switch (def.custom_def().custom_case()) {
+   case CustomOrderDefinition::kMarket:
+      return NativeOrderType::NATIVE_ORDER_TYPE_MARKET;
+      break;
+   case CustomOrderDefinition::kLimit:
+      return NativeOrderType::NATIVE_ORDER_TYPE_LIMIT;
+      break;
+   default:
+      break;
+   }
+   bts_assert(false);
+   return api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+}
+
+void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) {
+   c.key = key();
+   c.price = Price(me->state.price());
+   c.quantity =
+       bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity);
+   c.security = me->sec;
+   c.side = me->definition.side();
+   c.order_type = get_glbx_native_ord_type(me->definition);
+   c.time_in_force = me->definition.time_in_force();
+   c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+   c.manual_order_indicator = me->definition.manual_order_indicator();
+   c.routing_record = &me->routing_record;
+   LOG(OrderMan, Info, "GLBXNativeOrder::_create_create_msg c.qty: {}",
+       c.quantity);
+}
+
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/GLBXNativeOrder.h

@@ -11,6 +11,7 @@ class NewExecutionReportEvent;
 class CancelRejectEvent;
 class GatewayStatusChangeEvent;
 class ExecutionReport;
+struct CreateOrderMsg;
 
 class GLBXNativeOrder : public NativeOrderBase {
  public:
@@ -42,7 +43,7 @@ class GLBXNativeOrder : public NativeOrderBase {
    struct PImpl;
    std::unique_ptr<PImpl> me;
    void init_encodings();
-
+   void _create_create_msg(CreateOrderMsg &);
    void fixup_routing_record(api::orderman::RoutingRecord *rr);
    void on_exe_report(const NewExecutionReportEvent &);
    void on_cancel_reject(const CancelRejectEvent &);
@@ -60,8 +61,8 @@ class GLBXNativeOrder : public NativeOrderBase {
        log::Level,
        api::orderman::OrderStatus new_state,
        std::string const &msg);
-   void
-   transition(api::orderman::OrderStatus new_state, std::string const &msg);
+   void transition(
+       api::orderman::OrderStatus new_state, std::string const &msg);
    void transition(api::orderman::OrderStatus new_state);
    void transition_to_error(
        api::orderman::NativeOrderState::SubError sub_error,
@@ -74,8 +75,8 @@ class GLBXNativeOrder : public NativeOrderBase {
        api::orderman::NativeOrderState &,
        const NewExecutionReportEvent &) const;
    bool terminal_state(api::orderman::OrderStatus status) const;
-   api::orderman::NativeOrderState &
-   get_native_state(api::orderman::OrderState &);
+   api::orderman::NativeOrderState &get_native_state(
+       api::orderman::OrderState &);
    void ignore_or_cancel(
        std::string const &, api::orderman::OrderError, OrderOpResult *out);
 

src/server/sparkd/main.cpp

@@ -1,3 +1,4 @@
+#include "lib/md/cme/CMEActivSession.h"
 #include "lib/testing/catch_test_runner.h"
 #include "lib/utils/init_spark_lib.h"
 #include "server/sparkd/cmds/AdminReport.h"
@@ -10,6 +11,9 @@
 #include "server/sparkd/cmds/crypto/CryptoCmd.h"
 #include <CLI11/CLI11.hpp>
 
+#include <iostream>
+#include <string>
+
 using namespace bts;
 
 int main(int argc, char **argv) {
@@ -30,6 +34,9 @@ int main(int argc, char **argv) {
       app.add_subcommand(CLI::App_p(new cmds::Run()));
       app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
       app.add_subcommand(CLI::App_p(new cmds::Web()));
+
+     
+
       app.add_subcommand(
           "test", "Run unit test(s)"); // so this shows up in help
       CLI11_PARSE(app, argc, argv);