fix: changes spread_book_tick_sz from int to double to fix low tick being zero for reverse hedge orders

Bugs (fixed):
  * The SpreaderType dropdown only works for plain spreader orders. It
    doesn't work for icebergs, leans, etc.

.env

@@ -39,6 +39,7 @@ SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
 SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002

.vscode/launch.json

@@ -326,4 +326,4 @@
       "description": "Test to run",
     },
   ]
-}
+}

compose.yaml

@@ -33,6 +33,43 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
+  sparkd-symbols:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        ls -alh /home/bts/sparkdata/conf/
+        cat /home/bts/sparkdata/conf/.env.local
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
 
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet7

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -23,6 +23,7 @@
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
     <PackageReference Include="Parquet.Net" Version="5.0.1" />
     <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs

@@ -4,42 +4,46 @@ namespace bts.SilverPine {
    using System;
    using System.Collections.Generic;
    using System.Linq;
+   //using MathNet.Numerics.Statistics;
 
-   using ZstdSharp.Unsafe;
 
-   public enum GroupByBundleOp {
+   public enum MultiRowOp {
       Mean,
       Median,
       Mode,
+      Range,
       StdDev,
       Max,
       Min,
       First,
       Last,
-      Sum
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
    }
-   public class GroupByBundle {
-      public readonly List<Tuple<string, GroupByBundleOp>> operations;
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
 
-      public static GroupByBundle parse(string text) {
+      public static MultiRowOpHelper parse(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
-         List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
             string[] toks = c.Split(':');
             string inp_col = toks[0].Trim();
             string op_txt = toks[1].Trim();
-            GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
             rv.Add(Tuple.Create(inp_col, op));
          }
-         return new GroupByBundle(rv);
+         return new MultiRowOpHelper(rv);
       }
 
-      public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
          operations = operations_;
       }
 
-      public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
+      public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
          if (rows.Count() == 0) {
             return null;
          }
@@ -47,8 +51,8 @@ namespace bts.SilverPine {
          IBlueRow ir = rows.ElementAt(0);
          IBlueTableSchema in_schema = ir.schema;
          IBlueTableSchema out_schema = in_schema;
-         IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
-            new Dictionary<string, Tuple<string, GroupByBundleOp>>();
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
          foreach (var t in parameters.operations) {
             string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
             out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
@@ -58,7 +62,7 @@ namespace bts.SilverPine {
          foreach (var kv in out_col_map) {
             string out_col_name = kv.Key;
             string in_col_name = kv.Value.Item1;
-            GroupByBundleOp op = kv.Value.Item2;
+            MultiRowOp op = kv.Value.Item2;
             IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
             double comp_res = compute_op(col_vals, op);
             or.setValue(out_col_name, comp_res);
@@ -66,20 +70,51 @@ namespace bts.SilverPine {
          return or;
       }
 
-      public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
          switch (op) {
-            case GroupByBundleOp.Max:
+            case MultiRowOp.Max:
                return col_vals.Max<double>();
-            case GroupByBundleOp.Min:
+            case MultiRowOp.Min:
                return col_vals.Min<double>();
-            case GroupByBundleOp.Mean:
+            case MultiRowOp.Mean:
                return col_vals.Sum() / col_vals.Count();
-            case GroupByBundleOp.Sum:
+            case MultiRowOp.Sum:
                return col_vals.Sum();
-            case GroupByBundleOp.First:
+            case MultiRowOp.First:
                return col_vals.First();
-            case GroupByBundleOp.Last:
+            case MultiRowOp.Last:
                return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
             default:
                // TODO
                Log.assert(false);

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/AggregationSteam.cs

@@ -0,0 +1,20 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System;
+
+   [DataStreamUserRespresentation("AggregationStream")]
+   public class AgregationStream : ViewStream {
+
+      public AgregationStream(ViewStream parent_str, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, String.Empty, operation_txt);
+         mySetRealSource(rs);
+      }
+
+      public AgregationStream(ViewStream parent_str, string groupby_col, string operation_txt) {
+         ViewStream rs = new NonIncGroupByStream(parent_str, groupby_col, operation_txt);
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("aggregate")]
+   [DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;
@@ -53,15 +53,18 @@ namespace BCG {
                                  Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
                                  : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
 
-      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
-         GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
          Func<string[]> groupby_cols_func = () => {
             return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
          };
          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
             if (!String.IsNullOrEmpty(optional_sort_col))
                group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
-            IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
+            IBlueRow rv = MultiRowOpHelper.group_by(group_rows, std_group_by);
             return new IBlueRow[] { rv }.ToList();
          };
          ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -69,4 +69,6 @@ namespace bts {
          myFireEvent(dp_out);
       }
    }
+
+
 }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -13,9 +13,8 @@ namespace BCG {
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream

@@ -0,0 +1,14 @@
+namespace bts.SilverPine
+{
+   using BCG;
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream
+   {
+      private ViewStream _parent_str;
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz)
+      {
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Map/RollingComputationStream.cs

@@ -0,0 +1,10 @@
+namespace bts.SilverPine {
+   using BCG;
+
+
+   [DataStreamUserRespresentation("RollingComputationStream")]
+   public class RollingComputationStream : ViewStream {
+      public RollingComputationStream(ViewStream parent_str, string operation_txt, string roll_column, uint window_sz) {
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -108,6 +108,7 @@ namespace bts {
 
    }
 
+   [DataStreamUserRespresentation("Project")]
    public class NewProjectStream : ViewStream, IStreamMapper {
 
       private ViewStream _stream;

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -1,18 +1,15 @@
 using System;
 using System.Collections.Generic;
+
 using mbts;
 
-namespace BCG
-{
-   internal class MutableAppendTable : IBlueRowCollection
-   {
-      private class MyRow : IBlueRow
-      {
+namespace BCG {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
          private IBlueRow myRow;
          private int myID;
 
-         public MyRow(IBlueRow r, int id)
-         {
+         public MyRow(IBlueRow r, int id) {
             myRow = r;
             myID = id;
          }
@@ -21,10 +18,8 @@ namespace BCG
 
          public IBlueTableSchema schema { get { return myRow.schema; } }
 
-         public object this[int index]
-         {
-            get
-            {
+         public object this[int index] {
+            get {
                if (myRow.schema[index].colName == "id")
                   return myID;
                else
@@ -52,20 +47,17 @@ namespace BCG
 
          public System.Collections.ICollection Keys { get { return myRow.Keys; } }
 
-         public void Remove(object key) {throw new Exception("The method or operation is not implemented.");}
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
 
-         public System.Collections.ICollection Values
-         {
-            get
-            {
+         public System.Collections.ICollection Values {
+            get {
                object[] vals = new object[myRow.Count];
                CopyTo(vals, 0);
                return vals;
             }
          }
 
-         public object this[object key]
-         {
+         public object this[object key] {
             get { return key is string ? this[key as string] : this[(int)key]; }
             set { throw new Exception("The method or operation is not implemented."); }
          }
@@ -74,8 +66,7 @@ namespace BCG
 
          #region ICollection Members
 
-         public void CopyTo(Array array, int index)
-         {
+         public void CopyTo(Array array, int index) {
             myRow.CopyTo(array, index);
             array.SetValue(myID, index + myRow.schema["id"].colNo);
          }
@@ -90,8 +81,7 @@ namespace BCG
 
          #region IEnumerable Members
 
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-         { return new BlueRowEnumerator(this); }
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
 
          #endregion
       }
@@ -106,8 +96,7 @@ namespace BCG
 
       public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
 
-      public IBlueRow addRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
          if (!myIDMap.ContainsKey(tableNo))
             myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
          IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
@@ -116,20 +105,17 @@ namespace BCG
          return nr;
       }
 
-      public IBlueRow removeRow(IBlueRow r, int tableNo)
-      {
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
          IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
          myRows.deleteRow(dr);
          myIDMap[tableNo].Remove((int)r["id"]);
          return dr;
       }
 
-      public IBlueRowCollection removeTable(int tableNo)
-      {
+      public IBlueRowCollection removeTable(int tableNo) {
          if (!myIDMap.ContainsKey(tableNo))
             return null;
-         else
-         {
+         else {
             BlueTable rv = new BlueTable(myRows.schema);
             foreach (IBlueRow r in myIDMap[tableNo].Values)
                rv.addRow(r);
@@ -139,8 +125,7 @@ namespace BCG
          }
       }
 
-      public void clear()
-      {
+      public void clear() {
          myRows.clear();
          myIDMap.Clear();
       }
@@ -151,7 +136,7 @@ namespace BCG
 
       public IBlueTableSchema schema { get { return myRows.schema; } }
 
-      public bool containsID(int id) {return myRows.containsID(id);}
+      public bool containsID(int id) { return myRows.containsID(id); }
 
       #endregion
 
@@ -169,7 +154,7 @@ namespace BCG
 
       #region IEnumerable Members
 
-      public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator();}
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
 
       #endregion
    }
@@ -179,8 +164,8 @@ namespace BCG
    /// to append (or remove) more streams at
    /// runtime
    /// </summary>
-   public class MutableAppendStream : ViewStream
-   {
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -188,22 +173,17 @@ namespace BCG
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams))
-      { }
+         : this(new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams))
-      { }
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
 
       public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams)
-      { }
+         : this(true, streams) { }
 
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams)
-      {
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
          _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams)
-         {
+         foreach (ViewStream vs in streams) {
             myMovingIndex++;
             ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
             if (_allign_col_orders)
@@ -212,8 +192,7 @@ namespace BCG
          }
       }
 
-      public void addStream(ViewStream vs)
-      {
+      public void addStream(ViewStream vs) {
          myMovingIndex++;
          ///the change name is done to identify the callbacks, and the project to
          ///make all the schemas col orders the same
@@ -225,19 +204,14 @@ namespace BCG
             pj.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      public void removeStream(ViewStream vs)
-      {
+      public void removeStream(ViewStream vs) {
          int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-         {
-            if (p.Value.first == vs)
-            {
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
                index = p.Key;
-               if (mySubscriptionActive)
-               {
+               if (mySubscriptionActive) {
                   p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null)
-                  {
+                  if (myTable != null) {
                      IBlueRowCollection delRows = myTable.removeTable(p.Key);
                      if (delRows != null)
                         myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
@@ -250,15 +224,13 @@ namespace BCG
          }
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          mySubscriptionActive = true;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          mySubscriptionActive = false;
          foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
             p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
@@ -266,20 +238,17 @@ namespace BCG
             myTable.clear();
       }
 
-      private void myHandleStreamChanges(object newValue)
-      {         
+      private void myHandleStreamChanges(object newValue) {
          DataPacket dp = newValue as DataPacket;
          myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
          BlueTable adds = new BlueTable(myTable.schema),
                      dels = new BlueTable(myTable.schema);
          int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             IBlueRow dr = myTable.removeRow(r, index);
             dels.addRow(dr);
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             IBlueRow nr = myTable.addRow(r, index);
             adds.addRow(nr);
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -43,4 +43,11 @@ namespace bts.SilverPine {
          mySetRealSource(r);
       }
    }
+
+   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+   public class PeriodicVolumeAggregationStream : ViewStream {
+      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+                                             TimeSpan period, ViewStreamDatabase db) {
+      }
+   }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs

@@ -100,9 +100,9 @@ namespace bts.SilverPine {
       this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
 
       public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-         var fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt));
-
+         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+         // since this is already filtered, we can groupBy iit without a groupBy column
          r = new MathStream(r,
                  "vprice = volume * close");
          r = new NonIncGroupByStream(r,

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -90,20 +90,15 @@ namespace bts.SilverPine {
       // the argument is symbol, like "ES" etc and not the contract itself
       public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
       public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
          ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         // filtering for futures only
-         r = new FilterStream(r, r => {
-            string contract = r.get<string>("symbol");
-            var fc = new ContractUtils.FutureContract(contract);
-            return fc.future_tradable_contract != null;
-         });
-         r = new NonIncGroupByStream(r,
-                                      "ts_event",
-                                      "volume:Max");
-         r = new NewProjectStream(r,
-                                   "ts_event symbol !");
-         r = new SortStream(r,
-                             "-ts_event");
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
          mySetRealSource(r);
       }
    }

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs

@@ -2,15 +2,14 @@ namespace bts.SilverPine {
    using bts;
    using BCG;
 
-   // it uses the volume - better to use open interst perhap - to detect the rollover, is when a new 
-   // futuere exp becomes the highest volume one
+   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+   // future exp becomes the highest volume one
    [DataStreamUserRespresentation("FuturesRoll")]
    public class FuturesRollStream : ViewStream {
       public FuturesRollStream(string future_symbol) {
-         //ViewStream ohlcv_str = new OHLCVStream(future_symbol);
-         ViewStream rs = new FuturesActiveContractStream(future_symbol);
-         rs = new ColumnEvolutionStream(rs, "ts_event", "contract");
-         rs = new NewProjectStream(rs, "ts_event:roll_date !");
+         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
          mySetRealSource(rs);
       }
    }

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -2,21 +2,21 @@ namespace bts.SilverPine {
    using BCG;
 
    using bts;
-
+   // used directly in blueorb
    [DataStreamUserRespresentation("Settlement")]
    public class SettlementPriceStream : ViewStream {
 
-      public SettlementPriceStream(string future_name, string date_txt)
-      : this(future_name, date_txt, date_txt) { }
+      // public SettlementPriceStream(string future_name, string date_txt)
+      // : this(future_name, date_txt, date_txt) { }
 
-      public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-      : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
-      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt)
-      : this(symbol_or_contract, begin_dt, begin_dt) { }
+      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+      : this(future_contract, begin_dt, begin_dt) { }
 
-      public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract);
+      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
          ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
          rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
          mySetRealSource(rs);

src/DataStreamSuite/SharedStreams/NL/DSL_master_learnign_prompt.md

@@ -0,0 +1,276 @@
+# **DSL Master Learning Prompt**
+
+This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
+
+# **Introduction to Streams**
+
+The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
+
+# **Streams**
+
+## **FutureHistoricalBars (DataSource Stream)**
+
+The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
+
+#### **Stream arguments (parameters):**
+
+1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
+1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
+2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
+   For SIlver March April 2019 Silver Future Calendar spread)  
+3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
+2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
+   * `ONE_DAY` – Daily bars.  
+     * `ONE_MIN` – Minute bars.  
+3. **Optional:**  
+   * **`range_begin` (datetime):** Start of the desired time range.  
+   * **`range_end` (datetime):** End of the desired time range.
+
+#### **Schema:**
+
+The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
+
+| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
+| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
+| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
+|      |                       |          |         |         |         |         |        |
+
+# **FilterStream (Operator Stream)**
+
+Filters rows of a parent stream based on conditions applied to its columns.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
+2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
+
+#### **Examples:**
+
+1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
+2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
+3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
+   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
+   
+
+**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
+
+# **ProjectStream (Operator Stream)**
+
+Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
+
+#### **Stream arguments (parameters):**
+
+3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
+4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
+
+#### **Examples:**
+
+	The following DSL renames the open column to O, high to H, low to L and close to C  
+**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
+
+# **GroupByStream (Operator Stream)**
+
+The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
+2. **Group By Column (string, optional):**
+
+   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
+
+3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   Find the total SP 500 volume executed in the first week of 2021\.  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = GroupByStream(bars,”volume_Sum”)`  
+2. **NL:**   
+   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = GroupByStream(bars,”high:Max low:Min”)`  
+   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
+3. **NL:**   
+   **Find the Average daily volume per contract in Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
+   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
+
+# **SingleRowMathStream (Operator Stream)**
+
+The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **parent\_stream (another stream):**The source stream provides the base data  
+2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
+
+#### **Examples:**
+
+4. **NL:**   
+   Find the daily price movement in the SP 500 Futures through the month of Jan  
+   **DSL:**   
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
+   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
+5. **NL:**   
+   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
+
+#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
+
+**Schema Impact:**
+
+# **MultiRowRollingMathStream (Operator Stream)**
+
+The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
+
+---
+
+#### **Stream arguments (parameters):**
+
+1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
+2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
+   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
+3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
+4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
+
+---
+
+#### **Valid Aggregation Operations:**
+
+* `Mean`: Arithmetic mean of the values.  
+* `Median`: Median of the values.  
+* `Mode`: Most frequently occurring value.  
+* `Range`: Difference between the max and min.  
+* `StdDev`: Standard deviation of the values.  
+* `Max`: Maximum value.  
+* `Min`: Minimum value.  
+* `First`: First value in the group.  
+* `Last`: Last value in the group.  
+* `Sum`: Sum of the values.  
+* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
+* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+
+---
+
+#### **Examples:**
+
+1. **NL:**   
+   **Find the 20 d MA of SP 500 Futures close prices**  
+   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
+   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
+2. **NL:**   
+   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
+
+   **DSL:**   
+   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
+   `// calculating typical_price`
+
+   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
+   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
+
+   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
+
+3. **NL:**   
+   **Find the Average daily volume of Natural Gas in 2023**  
+   **DSL:**   
+   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
+   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
+   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+
+---
+
+#### **Schema Impact:**
+
+The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
+
+# **UnionStream (Operator Stream)**
+
+Like the SQL Union appends all the rows from 2 streams together.
+
+#### **Stream arguments (parameters):**
+
+5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
+6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
+
+#### **Examples:**
+
+4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
+   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
+   
+
+#### **Schema Impact:**
+
+None
+
+# **DateTimeExpanderStream (Operator Stream)**
+
+DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
+
+#### **Stream arguments (parameters):**
+
+1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
+2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
+
+**New Columns:**  
+`<input_dt_col>_date` (Date of the value, without the time part)
+
+* `<input_dt_col>_day,` (Day of the week)  
+* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
+* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
+* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
+* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
+* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
+
+#### **Examples:**
+
+**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
+**Input Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
+
+**Output Schema:**
+
+| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
+| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
+

src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json

@@ -2,9 +2,13 @@
    "prompts": [
       {
          "category": "Settlement",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however you should use the Today's real date",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or ESZ9 etc. The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for a range pf dates from begin_dt to end_dt. You will need to convert statements  expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August, First day of last quarter etc to actual dates. The function Settlement's second and third arguments are dates, so never pass a string like FirstDayOfLastQUarter. You will need to know what Today is, since there will be references to last week, yesterday etc.",
          "prompt": "What was the settlement for SP 500 Future on June 8th?",
          "examples": [
+            {
+               "natural_language": "What was the settlement for SP 500 Future on June 8th?",
+               "dsl": "Settlement(\"ES\", 6/8/2023)"
+            },
             {
                "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.",
                "dsl": "Settlement(\"ES\", 4/22/2023)"
@@ -41,7 +45,7 @@
       },
       {
          "category": "VWAP",
-         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
+         "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, period_start_date, period_end_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the start and end of the VWAP period, chronologically. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/05/2023  to 10/15/2023 (to cover for 10 days going back from the given date). so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt  will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ",
          "prompt": "What was the VWAP for SP 500 Future over the past 30 days?",
          "examples": [
             {
@@ -49,31 +53,39 @@
                "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)"
             },
             {
-               "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.",
-               "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)"
+               "natural_language": "Find the 10 day VWAP of the Crude Oil Future on 10/10/2023.",
+               "dsl": "VWAP(\"CL\", 10/01/2023, 10/10/2023)"
             },
             {
-               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.",
-               "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)"
+               "natural_language": "Find today's 30-day VWAP of the 10 year treasury future on 9/11/2023.",
+               "dsl": "VWAP(\"ZN\", 8/11/2023, 9/11/2023)"
             },
             {
                "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.",
                "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)"
+            },
+            {
+               "natural_language": "Find the 3 month VWAP of the SP 500 futures on 10/10/2023",
+               "dsl": "VWAP(\"ES\", 7/10/2023, 10/10/2023)"
             }
          ]
       },
       {
          "category": "FuturePriceDiff",
-         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
-         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?",
+         "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc)  please adjust the date to the nearest viable trading date (I am not providing any function to find this date, you have to find the nearest trading date yourself), for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest",
+         "prompt": "What is the difference in SP 500 Future from July 10th to July 14th? (or how much did the SP 500 Future price move from July 10th to July 14th?)",
          "examples": [
             {
                "natural_language": "What is the difference in ZN price from July 10th to July 14th?",
                "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)"
             },
             {
-               "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?",
-               "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)"
+               "natural_language": "What is the difference in SP 500 Future price from the first day of the first quarter of 2020 to the last day of the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
+            },
+            {
+               "natural_language": "How much did SP 500 Futures move in the first quarter of 2020?",
+               "dsl": "FuturePriceDiff(\"ES\", 1/1/2020, 3/31/2020)"
             },
             {
                "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?",
@@ -84,19 +96,19 @@
                "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)"
             },
             {
-               "natural_language": "What is the price move in  SP 500 Future during teh first quarter of last year",
+               "natural_language": "How much did  SP 500 Future move during the first quarter of last year",
                "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)"
             }
          ]
       },
       {
-         "category": "Aggregate Volume",
+         "category": "Contract Aggregate Volume",
          "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc).  The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.",
-         "prompt": "Which  SP 500 Future traded the most between 7/14/2024 and 7/10/2024",
+         "prompt": "Which  SP 500 Future traded the most between 7/10/2024 and 7/14/2024",
          "examples": [
             {
                "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024",
-               "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)"
+               "dsl": "VolumeAgg(\"ES\", \"Future\", 7/10/2024, 7/14/2024)"
             },
             {
                "natural_language": "Which Gold Spread traded the most in the first month of this year",
@@ -118,8 +130,8 @@
          "prompt": "When does the roll typically occur In  SP 500?",
          "examples": [
             {
-               "natural_language": "When does the roll typically occur In CL?",
-               "dsl": "FuturesRoll(\"CL\")"
+               "natural_language": "When does the roll typically occur In SP 500?",
+               "dsl": "FuturesRoll(\"ES\")"
             },
             {
                "natural_language": "When does the roll typically occur for 10 year treasury?",
@@ -163,8 +175,8 @@
          ]
       },
       {
-         "category": "Average Volume",
-         "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ",
+         "category": "Open Interest",
+         "description": "This is a few shot learning example for an NL to DSL.",
          "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?",
          "examples": [
             {

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -74,6 +74,11 @@ type TaifexExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
+type TwseExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type OkxExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -800,6 +800,7 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
     | bts.api.Venue.TAIFEX -> "TAIFEX"
+    | bts.api.Venue.TWSE -> "TWSE"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/client/spark/Modules/Spreader/ExecutionStrategyEditor.xaml

@@ -626,6 +626,22 @@
             </ComboBox.ItemTemplate>
           </ComboBox>
           <TextBlock
+            Grid.Column="6"
+            x:Name="PART_SpreaderType_Label"
+            VerticalAlignment="Center"
+            Margin="{StaticResource Margin.LHeaderPanelItem}"
+            Text="Type"
+          />
+          <ComboBox
+            x:Name="PART_SpreaderType_DropDown"
+            Grid.Column="7"
+            HorizontalAlignment="Stretch"
+            Margin="{StaticResource Margin.LHeaderPanelItem}"
+            ItemsSource="{TemplateBinding SpreaderTypeItemsSource}"
+            ItemTemplate="{StaticResource EnumDescriptionTemplate}"
+            SelectedItem="{Binding SpreaderType, RelativeSource={RelativeSource TemplatedParent}}"
+          />
+          <!-- <TextBlock
             Grid.Column="6"
             x:Name="PART_HedgePricing_Label"
             VerticalAlignment="Center"
@@ -642,7 +658,7 @@
             ItemTemplate="{StaticResource EnumDescriptionTemplate}"
             SelectedItem="{Binding HedgePricing, RelativeSource={RelativeSource TemplatedParent}}"
             Visibility="Collapsed"
-          />
+          /> -->
         </Grid>
         <c:DataGridEx
           x:Name="PART_DataGrid"

src/client/spark/Modules/Spreader/ExecutionStrategyEditor.xaml.cs

@@ -237,6 +237,29 @@ namespace mbts.ui.spark {
       }
       #endregion
 
+      #region SpreaderTypeItemsSource
+      public static readonly DependencyProperty SpreaderTypeItemsSourceProperty =
+         WPFHelper.create_dp<IEnumerable, ExecutionStrategyEditor>(
+            nameof(SpreaderTypeItemsSource));
+
+      public IEnumerable SpreaderTypeItemsSource {
+         get => GetValue(SpreaderTypeItemsSourceProperty) as IEnumerable;
+         set => SetValue(SpreaderTypeItemsSourceProperty, value);
+      }
+      #endregion
+
+      #region SpreaderType
+      public static readonly DependencyProperty SpreaderTypeProperty =
+         WPFHelper.create_dp<object, ExecutionStrategyEditor>(
+            nameof(SpreaderType),
+            FrameworkPropertyMetadataOptions.BindsTwoWayByDefault);
+
+      public object SpreaderType {
+         get => GetValue(SpreaderTypeProperty);
+         set => SetValue(SpreaderTypeProperty, value);
+      }
+      #endregion
+
       public override void OnApplyTemplate() =>
          _connect_leg_tuples();
 

src/client/spark/Modules/Spreader/SyntheticDataHelper.cs

@@ -117,6 +117,14 @@ namespace bts.shell {
          public static QuoteInsideMarketDistanceThrottle QuoteInsideMarketDistanceThrottle => new QuoteInsideMarketDistanceThrottle();
          #endregion
 
+         public static SniperQuoteParams SniperQuoteParams => new() {
+            NTicks = 0
+         };
+
+         public static ReverseHedgeQuoteParams ReverseHedgeQuoteParams => new() {
+            NTicks = 0
+         };
+
          #region PreHedge
          public static PreHedgeParams PreHedgeParams => new PreHedgeParams {
             MaxPriceFromInsideMarket = PreHedgeMaxPriceFromInsideMarket,

src/client/spark/Modules/Spreader/SyntheticEditorView.xaml

@@ -36,6 +36,8 @@
       ContractPickerVM="{Binding contract_picker_vm}"
       HedgePricing="{Binding hedge_pricing}"
       HedgePricingItemsSource="{Binding hedge_pricing_options}"
+      SpreaderType="{Binding spreader_type}"
+      SpreaderTypeItemsSource="{Binding spreader_type_options}"
       UseFractionalTicks="{Binding use_fractional_ticks}"
       IntegerTickSize="{Binding integer_tick_size}"
     />

src/client/spark/Modules/Spreader/VM/ParameterRows/Leg/EnableQuotingModel.cs

@@ -1,23 +0,0 @@
-using mbts.ui.spark;
-
-using System.Collections.Generic;
-using System.Linq;
-
-namespace mbts.spark {
-   partial class SyntheticEditorVM {
-      class IsQuoteModel : RowModelBase<bool> {
-         public override string display_name { get; } = "Enable Quoting";
-         public override string help_text { get; } = "This parameter is in the form of a check box, where if the box is checked the leg will be quoted versus not checked, it will not be shown in the market.  If it is not actively quoted in the market, the leg will still be used for calculation of the entire synthetic.";
-         public override string man_page { get; } = string.Empty;
-
-         public override bool get_value(LegSpreaderParamsTuple leg_tup) =>
-            leg_tup.leg_params.IsActiveQuoting;
-
-         public override void set_value(LegSpreaderParamsTuple leg_tup, bool value) =>
-            leg_tup.leg_params.IsActiveQuoting = value;
-
-         public override bool has_errors(IReadOnlyCollection<LegSpreaderParamsTuple> leg_tups) =>
-            !leg_tups.Any(lt => lt.leg_params.IsActiveQuoting);
-      }
-   }
-}

src/client/spark/Modules/Spreader/VM/ParameterRows/Quote/QuoteEnabledModel.cs

@@ -0,0 +1,25 @@
+using mbts.ui.spark;
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace mbts.spark;
+
+partial class SyntheticEditorVM {
+   class QuoteEnabledModel : RowModelBase<bool> {
+      public override string display_name => string.Empty;
+      public override string help_text => string.Empty;
+      public override string man_page => string.Empty;
+
+      public override ParameterSection[] section_path { get; } = new[] { Sections.Quote };
+
+      public override bool get_value(LegSpreaderParamsTuple leg_tup) =>
+         leg_tup.leg_params.IsActiveQuoting;
+
+      public override void set_value(LegSpreaderParamsTuple leg_tup, bool value) =>
+         leg_tup.leg_params.IsActiveQuoting = value;
+
+      public override bool has_errors(IReadOnlyCollection<LegSpreaderParamsTuple> leg_tups) =>
+         !leg_tups.Any(lt => lt.leg_params.IsActiveQuoting);
+   }
+}

src/client/spark/Modules/Spreader/VM/ParameterRows/ReverseHedge/ReverseHedgeEnabledModel.cs

@@ -0,0 +1,35 @@
+using bts.shell;
+
+using mbts.ui.spark;
+
+namespace mbts.spark;
+
+partial class SyntheticEditorVM {
+   class ReverseHedgeEnabledModel : RowModelBase<bool> {
+      readonly ReverseHedgeTicksModel _ticks_model;
+
+      public ReverseHedgeEnabledModel(ReverseHedgeTicksModel ticks_model) =>
+         _ticks_model = ticks_model;
+
+      public override string display_name => string.Empty;
+      public override string help_text => string.Empty;
+      public override string man_page => string.Empty;
+
+      public override ParameterSection[] section_path { get; } = new[] { Sections.ReverseHedge };
+
+      public override bool get_value(LegSpreaderParamsTuple leg_tup) =>
+         leg_tup.leg_params.ReverseHedgeQuoteParams is not null;
+
+      public override void set_value(LegSpreaderParamsTuple leg_tup, bool value) {
+         if (value) {
+            if (leg_tup.leg_params.ReverseHedgeQuoteParams is null) {
+               leg_tup.leg_params.ReverseHedgeQuoteParams = SyntheticDataHelper.Defaults.ReverseHedgeQuoteParams;
+               _ticks_model.apply_value(leg_tup);
+            }
+         } else {
+            leg_tup.leg_params.ReverseHedgeQuoteParams = null;
+         }
+         _ticks_model.set_enabled(leg_tup, value);
+      }
+   }
+}

src/client/spark/Modules/Spreader/VM/ParameterRows/ReverseHedge/ReverseHedgeTicksModel.cs

@@ -0,0 +1,42 @@
+using bts.spark;
+
+using mbts.ui.spark;
+
+using System.Collections.Generic;
+
+namespace mbts.spark;
+
+partial class SyntheticEditorVM {
+   internal class ReverseHedgeTicksModel : RowModelBase<int> {
+      readonly Dictionary<int, int> _value_store = new();
+
+      public override string display_name => "Ticks";
+      public override string help_text => string.Empty;
+      public override string man_page => string.Empty;
+
+      public override ParameterSection[] section_path { get; } = new[] { Sections.ReverseHedge };
+
+      public void apply_value(LegSpreaderParamsTuple leg_tup) =>
+         _get_params(leg_tup).NTicks = _value_from_store(leg_tup);
+
+      public override int get_value(LegSpreaderParamsTuple leg_tup) =>
+         _get_params(leg_tup)?.NTicks ?? _value_from_store(leg_tup);
+
+      public override void set_value(LegSpreaderParamsTuple leg_tup, int value) {
+         _value_store[leg_tup.id] = value;
+         var s = _get_params(leg_tup);
+         if (s is not null) {
+            s.NTicks = value;
+         }
+      }
+
+      protected override bool _get_enabled(LegSpreaderParamsTuple leg_tup) =>
+         _get_params(leg_tup) != null;
+
+      ReverseHedgeQuoteParams _get_params(LegSpreaderParamsTuple leg_tup) =>
+         leg_tup.leg_params.ReverseHedgeQuoteParams;
+
+      int _value_from_store(LegSpreaderParamsTuple leg_tup) =>
+         _value_store.get_value(leg_tup.id, _ => 0);
+   }
+}

src/client/spark/Modules/Spreader/VM/ParameterRows/RowModel.cs

@@ -94,7 +94,9 @@ namespace mbts.spark {
       }
 
       private static class Sections {
-         public static ParameterSection Quote => new ParameterSection("Quote");
+         public static ParameterSection Sniper => new("Sniper", true);
+         public static ParameterSection ReverseHedge => new("Reverse Hedge", true);
+         public static ParameterSection Quote => new("Quote", true);
          public static ParameterSection PreHedge => new ParameterSection("Pre-Hedge");
          public static ParameterSection PostHedge => new ParameterSection("Post-Hedge");
       }

src/client/spark/Modules/Spreader/VM/ParameterRows/Sniper/SniperEnabledModel.cs

@@ -0,0 +1,35 @@
+using bts.shell;
+
+using mbts.ui.spark;
+
+namespace mbts.spark;
+
+partial class SyntheticEditorVM {
+   class SniperEnabledModel : RowModelBase<bool> {
+      readonly SniperTicksModel _ticks_model;
+
+      public SniperEnabledModel(SniperTicksModel ticks_model) =>
+         _ticks_model = ticks_model;
+
+      public override string display_name => string.Empty;
+      public override string help_text => string.Empty;
+      public override string man_page => string.Empty;
+
+      public override ParameterSection[] section_path { get; } = new[] { Sections.Sniper };
+
+      public override bool get_value(LegSpreaderParamsTuple leg_tup) =>
+         leg_tup.leg_params.SniperQuoteParams is not null;
+
+      public override void set_value(LegSpreaderParamsTuple leg_tup, bool value) {
+         if (value) {
+            if (leg_tup.leg_params.SniperQuoteParams is null) {
+               leg_tup.leg_params.SniperQuoteParams = SyntheticDataHelper.Defaults.SniperQuoteParams;
+               _ticks_model.apply_value(leg_tup);
+            }
+         } else {
+            leg_tup.leg_params.SniperQuoteParams = null;
+         }
+         _ticks_model.set_enabled(leg_tup, value);
+      }
+   }
+}

src/client/spark/Modules/Spreader/VM/ParameterRows/Sniper/SniperTicksModel.cs

@@ -0,0 +1,42 @@
+using bts.spark;
+
+using mbts.ui.spark;
+
+using System.Collections.Generic;
+
+namespace mbts.spark;
+
+partial class SyntheticEditorVM {
+   internal class SniperTicksModel : RowModelBase<int> {
+      readonly Dictionary<int, int> _value_store = new();
+
+      public override string display_name => "Ticks";
+      public override string help_text => string.Empty;
+      public override string man_page => string.Empty;
+
+      public override ParameterSection[] section_path { get; } = new[] { Sections.Sniper };
+
+      public void apply_value(LegSpreaderParamsTuple leg_tup) =>
+         _get_params(leg_tup).NTicks = _value_from_store(leg_tup);
+
+      public override int get_value(LegSpreaderParamsTuple leg_tup) =>
+         _get_params(leg_tup)?.NTicks ?? _value_from_store(leg_tup);
+
+      public override void set_value(LegSpreaderParamsTuple leg_tup, int value) {
+         _value_store[leg_tup.id] = value;
+         var s = _get_params(leg_tup);
+         if (s is not null) {
+            s.NTicks = value;
+         }
+      }
+
+      protected override bool _get_enabled(LegSpreaderParamsTuple leg_tup) =>
+         _get_params(leg_tup) != null;
+
+      SniperQuoteParams _get_params(LegSpreaderParamsTuple leg_tup) =>
+         leg_tup.leg_params.SniperQuoteParams;
+
+      int _value_from_store(LegSpreaderParamsTuple leg_tup) =>
+         _value_store.get_value(leg_tup.id, _ => 0);
+   }
+}

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -23,6 +23,7 @@ using System.Windows.Input;
 using System.Windows.Media;
 
 using HedgePricing = bts.spark.ExecutionStrategy.Types.HedgePricing;
+using SpreaderType = bts.spark.ExecutionStrategy.Types.SpreaderType;
 
 namespace mbts.spark {
    /// <summary>
@@ -47,6 +48,7 @@ namespace mbts.spark {
       int _integer_tick_size = 1;
       ExecutionStrategyImmutable _strategy;
       HedgePricing _hedge_pricing;
+      SpreaderType _spreader_type;
 
       /// <summary>
       /// Inititalizes a new instance of SyntheticEditorVM class.  This constructor may only be used for already-existing synthetic securities (id must be non-zero).
@@ -198,6 +200,8 @@ namespace mbts.spark {
          } else {
             _can_add_leg = backing(nameof(can_add_leg), false);
          }
+         var sniper_ticks_model = new SniperTicksModel();
+         var reverse_hedge_ticks_model = new ReverseHedgeTicksModel();
          var show_quantity_quantity = new ShowQuantityQuantityModel();
 
          var quote_throttle_outside_ticks = new QuoteThrottleOutsideTicksModel();
@@ -225,15 +229,18 @@ namespace mbts.spark {
 
          var post_hedge_opposite_market_based_cross_threshold = new PostHedgeOppositeMarketVolumeBasedCrossThresholdModel();
          var post_hedge_opposite_market_based_cross_cross_ticks = new PostHedgeOppositeMarketVolumeBasedCrossCrossTicksModel();
-         var is_quote_model = new IsQuoteModel();
          rows = new IRowModel[] {
             _quantity_model,
             _price_mult_model,
-            is_quote_model,
             //new EnableHedgeModel(), //TAHA_SPREADER
             new HedgeRatioModel(),
+            new SniperEnabledModel(sniper_ticks_model),
+            sniper_ticks_model,
+            new ReverseHedgeEnabledModel(reverse_hedge_ticks_model),
+            reverse_hedge_ticks_model,
             //new MinLeanQtyModel(), //TAHA_SPREADER
             //new QuoteHiddenModel(),
+            new QuoteEnabledModel(),
             new QuoteQueueHolderModel(),
             new MarketDataTypeModel(),
             new ShowQuantityEnabledModel(show_quantity_quantity),
@@ -337,6 +344,13 @@ namespace mbts.spark {
       public HedgePricing hedge_pricing { get => _hedge_pricing; set => Set(ref _hedge_pricing, value); }
       public IEnumerable<HedgePricing> hedge_pricing_options => EnumHelper.get_values<HedgePricing>();
 
+      // spreader type
+      public SpreaderType spreader_type {
+         get => _spreader_type;
+         set => Set(ref _spreader_type, value);
+      }
+      public IEnumerable<SpreaderType> spreader_type_options => EnumHelper.get_values<SpreaderType>();
+
       public ExecutionStrategyImmutable get_strategy() {
          var mutable_strategy = _strategy.CloneToMutable();
          mutable_strategy.LegParamsMap.Clear();
@@ -352,6 +366,7 @@ namespace mbts.spark {
          _synthetic.Exchange = Venue.Bts;
          _strategy = strategy;
          hedge_pricing = strategy.HedgePricing;
+         spreader_type = strategy.SpreaderType;
          leg_tuples.SetItems(_get_legs(_synthetic, strategy));
          if (synthetic.HasContractSpec) {
             /// assign the fields <see cref="_use_fractional_ticks"/> and <see cref="_integer_tick_size"/>,

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -41,6 +41,8 @@ namespace mbts.spark {
    using System.Windows.Input;
    using System.Windows.Media;
 
+   using SpreaderType = bts.spark.ExecutionStrategy.Types.SpreaderType;
+
    class LadderVM : DisposableViewModelBase, IWillLetYouKnowWhenIDispose, IKeyListener {
       static readonly Brush _transparent_status_brush = BTSColors.UI.transparent;
 
@@ -103,6 +105,7 @@ namespace mbts.spark {
       ulong _misc_time = 0;
       ulong _last_trade_time = 0;
       ulong _last_bbo_time = 0;
+      SpreaderType _spreader_type = SpreaderType.Requote;
       IReadOnlyDictionary<uint, OrderRecordTuple> _cache = DictionaryHelper.empty<uint, OrderRecordTuple>();
       ImmutableDictionary<AlgoKey, LadderFavoriteAlgoMetadata> _favorite_algo_metadata_map =
          ImmutableDictionary<AlgoKey, LadderFavoriteAlgoMetadata>.Empty;
@@ -126,6 +129,9 @@ namespace mbts.spark {
          size_calculator.display_settings = model.security_md.quantity_display;
          _slot_price_vs_orders = new Dictionary<double, LadderOrderVM>(_equality);
          this.order_properties_vm = order_properties_vm;
+         if (model.security.HasSynthetic) {
+            spreader_type_choices.AddRange(EnumHelper.get_values<SpreaderType>());
+         }
          _session_high = backing(nameof(session_high), double.NaN);
          _session_low = backing(nameof(session_low), double.NaN);
          _day_change = backing(nameof(day_change), double.NaN);
@@ -578,6 +584,12 @@ namespace mbts.spark {
       public ObservableCollection<FairValueLineVM> fair_value_lines { get; } =
          new ObservableCollection<FairValueLineVM>();
 
+      public SpreaderType spreader_type {
+         get => _spreader_type;
+         set => Set(ref _spreader_type, value);
+      }
+      public ObservableCollection<SpreaderType> spreader_type_choices { get; } = new();
+
       public uint orders_column_width {
          get => _orders_column_width.Value;
          set => _orders_column_width.set(value);
@@ -757,15 +769,21 @@ namespace mbts.spark {
          }
 
          var order = order_properties_vm.create(side);
-         if (order.CustomDef is not null) {
-            if (order.CustomDef.Stop is not null) {
-               order.CustomDef.Stop.StopPrice = price;
-            } else if (order.CustomDef.StopLimit is not null) {
+         var def = order.CustomDef;
+         if (def is not null) {
+            if (def.Stop is not null) {
+               def.Stop.StopPrice = price;
+            } else if (def.StopLimit is not null) {
                // swap the prices here so that the level clicked
                // in the ladder is the StopPrice (trigger) and the Price
                // is what was specificed in the extra parameter box
-               (price, order.CustomDef.StopLimit.StopPrice) =
-                  (order.CustomDef.StopLimit.StopPrice, price);
+               (price, def.StopLimit.StopPrice) =
+                  (def.StopLimit.StopPrice, price);
+            }
+            if (security.HasSynthetic) {
+               foreach (var d in OrderDataHelper.enumerate_algo_hierarchy(def).Where(d => d.CustomCase == CustomOrderDefinition.CustomOneofCase.Spreader)) {
+                  d.Spreader.ExecutionStrategy.SpreaderType = spreader_type;
+               }
             }
          }
 

src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml

@@ -956,6 +956,13 @@
                 </Style>
               </uag:OrderDetailsPopupControl.Resources>
             </uag:OrderDetailsPopupControl>
+            
+            <ComboBox
+              Margin="2"
+              ItemsSource="{Binding spreader_type_choices}"
+              SelectedItem="{Binding spreader_type}"
+              Visibility="{Binding HasItems, Mode=OneWay, RelativeSource={RelativeSource Self}, Converter={StaticResource BooleanToVisibilityConverter}}"
+            />
 
             <!-- Position into Qty Button -->
             <Button

src/client/spark/Modules/Trading/PNL/Data/PNLNormalizedData.cs

@@ -164,6 +164,8 @@ namespace bts.shell.pnl {
          new() {
             // sod
             sod_pnl = op(a.sod_pnl, b.sod_pnl),
+            sod_realized_pnl = op(a.sod_realized_pnl, b.sod_realized_pnl),
+            sod_unrealized_pnl = op(a.sod_unrealized_pnl, b.sod_unrealized_pnl),
             sod_long_pos = op(a.sod_long_pos, b.sod_long_pos),
             sod_short_pos = op(a.sod_short_pos, b.sod_short_pos),
             sod_delta = op(a.sod_delta, b.sod_delta),
@@ -175,6 +177,8 @@ namespace bts.shell.pnl {
 
             // day
             day_pnl = op(a.day_pnl, b.day_pnl),
+            day_realized_pnl = op(a.day_realized_pnl, b.day_realized_pnl),
+            day_unrealized_pnl = op(a.day_unrealized_pnl, b.day_unrealized_pnl),
             day_long_pos = op(a.day_long_pos, b.day_long_pos),
             day_short_pos = op(a.day_short_pos, b.day_short_pos),
             day_delta = op(a.day_delta, b.day_delta),

src/client/spark/Modules/Trading/PNL/PNLWindow.cs

@@ -39,6 +39,9 @@ namespace bts.shell {
    class PNLWindow : SparkWindow<PNLSaveState>, ISharedWindow {
       public const string WINDOW_TYPE = "PNL";
       public const string WINDOW_TITLE = "P&L Summary";
+      // Cpp overrides aren't respected in pnl calc. Hide the "Override Cpp"
+      // menu item until we fix it.
+      const bool CAN_OVERRIDE_CPP = false;
 
       static int _window_id_counter = 0;
 
@@ -156,7 +159,7 @@ namespace bts.shell {
             },
             new SparkCommandCommand(
                "Override CPP...", Icons.CurrencyExchange, new Command(_override_cpp)) {
-               show_in_context_menu = true,
+               show_in_context_menu = CAN_OVERRIDE_CPP,
                show_in_toolbar = false
             },
             SparkCommands.window_settings_command(new DialogCommand(_mk_settings_win)),

src/client/spark/Modules/Trading/PNL/Settings/PNLSettings.cs

@@ -225,16 +225,28 @@ namespace bts.shell.pnl {
          get {
             yield return _trigger($"{nameof(PNLNormalizedData.sod_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.SODPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.sod_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.SODPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_realized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.SODRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_realized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.SODRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_unrealized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.SODUnrealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_unrealized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.SODUnrealizedPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.sod_net_pos)} > 0", _animation(foreground: BTSColors.MarketData.bid, column: Columns.SODNetPos));
             yield return _trigger($"{nameof(PNLNormalizedData.sod_net_pos)} < 0", _animation(foreground: BTSColors.MarketData.ask, column: Columns.SODNetPos));
             // day
             yield return _trigger($"{nameof(PNLNormalizedData.day_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.DayPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.day_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.DayPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_realized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.DayRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_realized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.DayRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_unrealized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.DayUnrealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_unrealized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.DayUnrealizedPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.day_net_pos)} > 0", _animation(foreground: BTSColors.MarketData.bid, column: Columns.DayNetPos));
             yield return _trigger($"{nameof(PNLNormalizedData.day_net_pos)} < 0", _animation(foreground: BTSColors.MarketData.ask, column: Columns.DayNetPos));
             // total
             yield return _trigger($"{nameof(PNLNormalizedData.total_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.TotalPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.total_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.TotalPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_realized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.TotalRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_realized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.TotalRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_unrealized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.TotalUnrealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_unrealized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.TotalUnrealizedPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.total_net_pos)} > 0", _animation(foreground: BTSColors.MarketData.bid, column: Columns.TotalNetPos));
             yield return _trigger($"{nameof(PNLNormalizedData.total_net_pos)} < 0", _animation(foreground: BTSColors.MarketData.ask, column: Columns.TotalNetPos));
             // working
@@ -247,16 +259,28 @@ namespace bts.shell.pnl {
          get {
             yield return _trigger($"{nameof(PNLNormalizedData.sod_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.SODPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.sod_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.SODPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_realized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.SODRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_realized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.SODRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_unrealized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.SODUnrealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.sod_unrealized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.SODUnrealizedPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.sod_net_pos)} > 0", _animation(foreground: BTSColors.MarketData.bid, column: Columns.SODNetPos));
             yield return _trigger($"{nameof(PNLNormalizedData.sod_net_pos)} < 0", _animation(foreground: BTSColors.MarketData.ask, column: Columns.SODNetPos));
             // day
             yield return _trigger($"{nameof(PNLNormalizedData.day_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.DayPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.day_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.DayPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_realized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.DayRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_realized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.DayRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_unrealized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.DayUnrealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.day_unrealized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.DayUnrealizedPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.day_net_pos)} > 0", _animation(foreground: BTSColors.MarketData.bid, column: Columns.DayNetPos));
             yield return _trigger($"{nameof(PNLNormalizedData.day_net_pos)} < 0", _animation(foreground: BTSColors.MarketData.ask, column: Columns.DayNetPos));
             // total
             yield return _trigger($"{nameof(PNLNormalizedData.total_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.TotalPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.total_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.TotalPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_realized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.TotalRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_realized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.TotalRealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_unrealized_pnl)} > 0", _animation(foreground: BTSColors.PNL.profit, column: Columns.TotalUnrealizedPNL));
+            yield return _trigger($"{nameof(PNLNormalizedData.total_unrealized_pnl)} < 0", _animation(foreground: BTSColors.PNL.loss, column: Columns.TotalUnrealizedPNL));
             yield return _trigger($"{nameof(PNLNormalizedData.total_net_pos)} > 0", _animation(foreground: BTSColors.MarketData.bid, column: Columns.TotalNetPos));
             yield return _trigger($"{nameof(PNLNormalizedData.total_net_pos)} < 0", _animation(foreground: BTSColors.MarketData.ask, column: Columns.TotalNetPos));
             // working

src/client/spark/Modules/Trading/PNL/VM/PNLHeaderRowVM.cs

@@ -34,6 +34,8 @@ namespace bts.shell.pnl {
          _cell_map.set_appearances(default_appearances);
          // sod
          sod_pnl = _cell_map.add_pnl_decimal(ColumnKeys.SODPNL, d => d.sod_pnl);
+         sod_realized_pnl = _cell_map.add_pnl_decimal(ColumnKeys.SODRealizedPNL, d => d.sod_realized_pnl);
+         sod_unrealized_pnl = _cell_map.add_pnl_decimal(ColumnKeys.SODUnrealizedPNL, d => d.sod_unrealized_pnl);
          sod_net_pos = _cell_map.add_net_pos(ColumnKeys.SODNetPos, d => d.sod_net_pos);
          sod_long_pos = _cell_map.add_long_pos(ColumnKeys.SODLongPos, d => d.sod_long_pos);
          sod_short_pos = _cell_map.add_short_pos(ColumnKeys.SODShortPos, d => d.sod_short_pos);
@@ -44,6 +46,8 @@ namespace bts.shell.pnl {
 
          // day
          day_pnl = _cell_map.add_pnl(ColumnKeys.DayPNL, d => d.day_pnl);
+         day_realized_pnl = _cell_map.add_pnl_decimal(ColumnKeys.DayRealizedPNL, d => d.day_realized_pnl);
+         day_unrealized_pnl = _cell_map.add_pnl_decimal(ColumnKeys.DayUnrealizedPNL, d => d.day_unrealized_pnl);
          day_net_pos = _cell_map.add_net_pos(ColumnKeys.DayNetPos, d => d.day_net_pos);
          day_long_pos = _cell_map.add_long_pos(ColumnKeys.DayLongPos, d => d.day_long_pos);
          day_short_pos = _cell_map.add_short_pos(ColumnKeys.DayShortPos, d => d.day_short_pos);
@@ -58,6 +62,8 @@ namespace bts.shell.pnl {
          working_short_pos = _cell_map.add_short_pos(ColumnKeys.WorkingShortPos, r => r.working_short_pos);
          // total
          total_pnl = _cell_map.add_pnl_decimal(ColumnKeys.TotalPNL, d => d.total_pnl);
+         total_realized_pnl = _cell_map.add_pnl_decimal(ColumnKeys.TotalRealizedPNL, d => d.total_realized_pnl);
+         total_unrealized_pnl = _cell_map.add_pnl_decimal(ColumnKeys.TotalUnrealizedPNL, d => d.total_unrealized_pnl);
          total_net_pos = _cell_map.add_net_pos(ColumnKeys.TotalNetPos, d => d.total_net_pos);
          total_long_pos = _cell_map.add_long_pos(ColumnKeys.TotalLongPos, d => d.total_long_pos);
          total_short_pos = _cell_map.add_short_pos(ColumnKeys.TotalShortPos, d => d.total_short_pos);
@@ -88,6 +94,8 @@ namespace bts.shell.pnl {
 
       // day
       public PNLCellVM<PNLNormalizedData, decimal> day_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> day_realized_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> day_unrealized_pnl { get; }
       public PNLCellVM<PNLNormalizedData, decimal> day_long_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> day_short_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> day_net_pos { get; }
@@ -98,6 +106,8 @@ namespace bts.shell.pnl {
 
       // sod
       public PNLCellVM<PNLNormalizedData, decimal> sod_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> sod_realized_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> sod_unrealized_pnl { get; }
       public PNLCellVM<PNLNormalizedData, decimal> sod_long_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> sod_short_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> sod_net_pos { get; }
@@ -108,6 +118,8 @@ namespace bts.shell.pnl {
 
       // total
       public PNLCellVM<PNLNormalizedData, decimal> total_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> total_realized_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> total_unrealized_pnl { get; }
       public PNLCellVM<PNLNormalizedData, decimal> total_net_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> total_long_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> total_short_pos { get; }

src/client/spark/Modules/Trading/PNL/View/PNLRowCollectionView.cs

@@ -118,6 +118,8 @@ namespace bts.shell.pnl {
          });
          // sod
          sod_pnl = _cell_map.add_pnl(ColumnKeys.SODPNL, d => d.sod_pnl);
+         sod_realized_pnl = _cell_map.add_pnl(ColumnKeys.SODRealizedPNL, d => d.sod_realized_pnl);
+         sod_unrealized_pnl = _cell_map.add_pnl(ColumnKeys.SODUnrealizedPNL, d => d.sod_unrealized_pnl);
          sod_net_pos = _cell_map.add_net_pos(ColumnKeys.SODNetPos, d => d.display_sod_net_pos);
          sod_long_pos = _cell_map.add_long_pos(ColumnKeys.SODLongPos, d => d.display_sod_long_pos);
          sod_short_pos = _cell_map.add_short_pos(ColumnKeys.SODShortPos, d => d.display_sod_short_pos);
@@ -128,6 +130,8 @@ namespace bts.shell.pnl {
 
          // day
          day_pnl = _cell_map.add_pnl(ColumnKeys.DayPNL, d => d.day_pnl);
+         day_realized_pnl = _cell_map.add_pnl(ColumnKeys.DayRealizedPNL, d => d.day_realized_pnl);
+         day_unrealized_pnl = _cell_map.add_pnl(ColumnKeys.DayUnrealizedPNL, d => d.day_unrealized_pnl);
          day_net_pos = _cell_map.add_net_pos(ColumnKeys.DayNetPos, d => d.display_day_net_pos);
          day_long_pos = _cell_map.add_long_pos(ColumnKeys.DayLongPos, d => d.display_day_long_pos);
          day_short_pos = _cell_map.add_short_pos(ColumnKeys.DayShortPos, d => d.display_day_short_pos);
@@ -138,6 +142,8 @@ namespace bts.shell.pnl {
 
          // total
          total_pnl = _cell_map.add(ColumnKeys.TotalPNL, d => d.total_pnl, FormattingDelegates.Price<decimal>.Value);
+         total_realized_pnl = _cell_map.add_pnl(ColumnKeys.TotalRealizedPNL, d => d.total_realized_pnl);
+         total_unrealized_pnl = _cell_map.add_pnl(ColumnKeys.TotalUnrealizedPNL, d => d.total_unrealized_pnl);
          total_net_pos = _cell_map.add(ColumnKeys.TotalNetPos, d => d.display_total_net_pos);
          total_long_pos = _cell_map.add(ColumnKeys.TotalLongPos, d => d.display_total_long_pos);
          total_short_pos = _cell_map.add(ColumnKeys.TotalShortPos, d => d.display_total_short_pos);
@@ -181,6 +187,8 @@ namespace bts.shell.pnl {
 
       // sod
       public PNLCellVM<PNLNormalizedData, decimal> sod_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> sod_realized_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> sod_unrealized_pnl { get; }
       public PNLCellVM<PNLNormalizedData, decimal> sod_net_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> sod_long_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> sod_short_pos { get; }
@@ -191,6 +199,8 @@ namespace bts.shell.pnl {
 
       // day
       public PNLCellVM<PNLNormalizedData, decimal> day_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> day_realized_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> day_unrealized_pnl { get; }
       public PNLCellVM<PNLNormalizedData, decimal> day_net_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> day_long_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> day_short_pos { get; }
@@ -201,6 +211,8 @@ namespace bts.shell.pnl {
 
       // total
       public PNLCellVM<PNLNormalizedData, decimal> total_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> total_realized_pnl { get; }
+      public PNLCellVM<PNLNormalizedData, decimal> total_unrealized_pnl { get; }
       public PNLCellVM<PNLNormalizedData, decimal> total_net_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> total_long_pos { get; }
       public PNLCellVM<PNLNormalizedData, decimal> total_short_pos { get; }

src/client/spark/Modules/Trading/PNL/View/PNLView.xaml

@@ -266,6 +266,7 @@
         <Setter Property="VerticalAlignment" Value="Center"/>
         <Setter Property="Margin" Value="4,0,0,0"/>
         <Setter Property="FontWeight" Value="Bold"/>
+        <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}"/>
       </Style>
 
       <Style x:Key="HeaderRowCellText" TargetType="TextBlock">

src/client/spark/Orders/OrderFactory.cs

@@ -143,10 +143,10 @@ namespace mbts.spark.utilities {
          bool is_child = false
       ) {
          var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
-         if (order_type != OrderType.Limit) {
-            tifs = tifs.Except(new[] { TimeInForce.Ioc });
-         }
-         return tifs;
+         return order_type switch {
+            OrderType.Limit or OrderType.Iceberg => tifs,
+            _ => tifs.Except(TimeInForce.Ioc),
+         };
       }
 
       public IEnumerable<string> dynamic_types() =>

src/client/spark_api/security/SecurityExtensions.cs

@@ -123,6 +123,7 @@ namespace bts.api {
             Security.ExchangeDataOneofCase.Okx => security.Okx.SecurityDesc,
             Security.ExchangeDataOneofCase.Cboe => security.Cboe.SecurityDesc,
             Security.ExchangeDataOneofCase.Taifex => security.Taifex.SecurityDesc,
+            Security.ExchangeDataOneofCase.Twse => security.Twse.SecurityDesc,
             _ => throw new NotImplementedException("security_desc not implemented for exchange")
          };
 
@@ -173,6 +174,7 @@ namespace bts.api {
             domain.Venue.Okx => Venue.Okx,
             domain.Venue.Cboe => Venue.Cboe,
             domain.Venue.Taifex => Venue.Taifex,
+            domain.Venue.Twse => Venue.Twse,
 
 
             domain.Venue.Unspecified => Venue.Unset,
@@ -194,6 +196,7 @@ namespace bts.api {
             Venue.Okx => domain.Venue.Okx,
             Venue.Cboe => domain.Venue.Cboe,
             Venue.Taifex => domain.Venue.Taifex,
+            Venue.Twse => domain.Venue.Twse,
 
             Venue.Unset => domain.Venue.Unspecified,
             _ => throw new NotImplementedException(),

src/lib/api/domain/Venue.proto

@@ -27,4 +27,5 @@ enum Venue {
   VENUE_OKX = 15;
   VENUE_TAIFEX = 16;
   VENUE_CBOE = 17;
+  VENUE_TWSE = 18;
 }

src/lib/api/md/MarketDataConfig.proto

@@ -13,6 +13,7 @@ import "lib/api/md/EmumbaDeribitMarketDataConfig.proto";
 import "lib/api/md/OkxMarketDataConfig.proto";
 import "lib/api/md/GlbxMarketDataConfig.proto";
 import "lib/api/md/TaifexMarketDataConfig.proto";
+import "lib/api/md/TwseMarketDataConfig.proto";
 import "lib/api/md/CboeMarketDataConfig.proto";
 
 option csharp_namespace = "bts.api.md";
@@ -31,6 +32,7 @@ message MarketDataConfig {
   repeated GlbxMarketDataConfig glbx = 8;
   repeated CboeMarketDataConfig cboe = 9;
   repeated TaifexMarketDataConfig taifex =10;
+  repeated TwseMarketDataConfig twse = 12;
   // Add an entry here for all the exchanges that need their market data to be recorded
 
   repeated EmumbaDeribitMarketDataConfig emumba_deribit = 100;

src/lib/api/md/MarketDataConfigUtils.cpp

@@ -23,6 +23,8 @@ std::set<bts::api::Venue> MarketDataConfigUtils::get_live_exchanges(
       rv.insert(bts::api::VENUE_GLBX);
    if (cfg.taifex_size()>0)
       rv.insert(bts::api::VENUE_TAIFEX);
+   if (cfg.twse_size()>0)
+      rv.insert(bts::api::VENUE_TWSE);
    if (cfg.cboe_size()>0)
       rv.insert(bts::api::VENUE_CBOE);
    return rv;

src/lib/api/md/TwseMarketDataConfig.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+import "lib/api/domain/SecurityCriteria.proto";
+
+package bts.api.md;
+
+option csharp_namespace = "bts.api.md";
+message TwseConnectionDetails{
+  string live_data_ip = 30;
+}
+
+message TwseMarketDataConfig {
+  uint32 unique_check_value = 1;
+  bool read_from_file = 20;
+  string realtime_data_file_name = 22;
+  TwseConnectionDetails connection_details = 30;
+  bts.api.domain.SecurityCriteria security_criteria = 3;
+  
+}

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -73,6 +73,7 @@ bool AdminServiceUtils::has_exchange(
    case api::Venue::VENUE_OKX:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -189,6 +190,7 @@ AdminServiceUtils::resolve_connection(
    case api::Venue::VENUE_BINANCE_COIN_FUTURES:
    case api::Venue::VENUE_BINANCE_US:
    case api::Venue::VENUE_COINBASE:
+   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOE:
    case api::Venue::VENUE_BINANCE_USDS_FUTURES: {
@@ -463,6 +465,7 @@ std::string AdminServiceUtils::find_account_exchange_account(
    case api::Venue::VENUE_OKX:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -787,6 +790,7 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
    case api::Venue::VENUE_COINBASE:
    case api::Venue::VENUE_UNSET:
    case api::Venue::VENUE_BTS:
+   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
    case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
@@ -1183,6 +1187,7 @@ std::string AdminServiceUtils::find_user_exchange_id(
    case api::Venue::VENUE_UNSET:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
    case api::Venue::VENUE_CBOECFE:
    case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
@@ -1222,6 +1227,7 @@ std::string AdminServiceUtils::find_user_company_id(
    case api::Venue::VENUE_UNSET:
    case api::Venue::VENUE_BTS:
    case api::Venue::VENUE_CBOE:
+   case api::Venue::VENUE_TWSE:
    case api::Venue::VENUE_TAIFEX:
 
    case api::Venue::VENUE_CBOECFE:

src/lib/config/profiles/default/symbols.conf

@@ -24,6 +24,14 @@ SymbolLists {
       symbol: "BTC"
       symbol: "ETH"
     }
+    list {
+      key: "TWSE"
+      symbol: "2330"
+      symbol: "2317"
+      symbol: "2454"
+      symbol: "2603"
+      symbol: "2615"
+    }
     list {
       key: "TAIFEX"
       symbol: "QFFH"
@@ -65,6 +73,11 @@ Symbols {
       future: @SymbolLists.list[GLBX].symbol
       spread: @SymbolLists.list[GLBX].symbol
     }
+    venues{
+      name: "VENUE_TWSE"
+      future: @SymbolLists[TWSE].symbol
+      option: @SymbolLists[TWSE].symbol
+    }
     venues{
       name: "VENUE_TAIFEX"
       future: @SymbolLists[TAIFEX].symbol
@@ -97,6 +110,11 @@ Symbols {
       future: @SymbolLists.list[ICE].symbol
       spread: @SymbolLists.list[ICE].symbol
     }
+    venues{
+      name: "VENUE_TWSE"
+      future: @SymbolLists[TWSE].symbol
+      option: @SymbolLists[TWSE].symbol
+    }
     venues{
       name: "VENUE_TAIFEX"
       future: @SymbolLists[TAIFEX].symbol
@@ -131,6 +149,11 @@ Symbols {
       option: @SymbolLists.list[DERIBIT].symbol
       spread: @SymbolLists.list[DERIBIT].symbol
     }
+    venues{
+      name: "VENUE_TWSE"
+      future: @SymbolLists.list[TWSE].symbol
+      option: @SymbolLists.list[TWSE].symbol
+    }
     venues{
       name: "VENUE_TAIFEX"
       future: @SymbolLists.list[TAIFEX].symbol

src/lib/domain/DomainSecurityUtils.cpp

@@ -23,6 +23,8 @@ api::domain::Venue DomainSecurityUtils::convert(bts::api::Venue venue) {
       return DOMAIN_VENUE(DERIBIT);
    case bts::api::Venue::VENUE_CBOE:
       return DOMAIN_VENUE(CBOE);
+   case bts::api::Venue::VENUE_TWSE:
+      return DOMAIN_VENUE(TWSE);
    case bts::api::Venue::VENUE_TAIFEX:
       return DOMAIN_VENUE(TAIFEX);
    case bts::api::Venue::VENUE_BTS:
@@ -59,6 +61,8 @@ bts::api::Venue DomainSecurityUtils::convert(api::domain::Venue venue) {
       return api::Venue::VENUE_DERIBIT;
    case DOMAIN_VENUE(CBOE):
       return api::Venue::VENUE_CBOE;
+   case DOMAIN_VENUE(TWSE):
+      return api::Venue::VENUE_TWSE;
    case DOMAIN_VENUE(TAIFEX):
       return api::Venue::VENUE_TAIFEX;
    case DOMAIN_VENUE(FTX):
@@ -171,6 +175,9 @@ bool DomainSecurityUtils::get_security_desc(
    case bts::api::Security::ExchangeDataCase::kCboe:
       desc = security.cboe().security_desc();
       break;
+   case bts::api::Security::ExchangeDataCase::kTwse:
+      desc = security.twse().security_desc();
+      break;
    case bts::api::Security::ExchangeDataCase::kTaifex:
       desc = security.taifex().security_desc();
       break;

src/lib/domain_model/Security.h

@@ -346,6 +346,8 @@ inline const std::string &descriptor(
       return sec.coinbase().security_desc();
    case api::Security::kOkx:
       return sec.okx().security_desc();
+   case api::Security::kTwse:
+      return sec.twse().security_desc();
    case api::Security::kTaifex:
       return sec.taifex().security_desc();
    default: {

src/lib/domain_model/Security.proto

@@ -21,6 +21,7 @@ enum Venue {
      VENUE_COINBASE = 450;
      VENUE_OKX = 460;
      VENUE_TAIFEX = 500;
+     VENUE_TWSE = 600;
 }
 
 enum SecurityType {
@@ -88,6 +89,11 @@ message CBOEExchangeData {
     int32 class_key = 30;
     string source = 40;
 }
+message TWSEExchangeData {
+    int64 exchange_id = 10;
+    int32 class_key = 30;
+    string security_desc = 20;
+}
 message TAIFEXExchangeData {
     int64 exchange_id = 10;
     int32 class_key = 30;
@@ -260,6 +266,7 @@ message Security {
        CoinbaseExchangeData coinbase = 290;
        OkxExchangeData okx = 300;
        TAIFEXExchangeData taifex = 400;
+       TWSEExchangeData twse = 500;
    }
 }
 message SecurityList {

src/lib/md/GeneralMarketDataListener.cpp

@@ -19,6 +19,8 @@
 #include "lib/domain/SpreadSecurity.h"
 #include "lib/domain/TradeRecord.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
+#include "lib/io/JsonDumper.h"
 #include "lib/logging/logging.h"
 #include "lib/md/GeneralSecurityIndex.h"
 #include "lib/md/IMarketDataListener.h"
@@ -129,6 +131,16 @@ struct IncrementalBookData {
             int insert_position = 0;
             for (size_t i = 0; i < book_side.size();
                  i++) { //not sure how to access book
+               if (book_side[i].price == op.price) {
+                  IMarketDataListener::BookOp new_op;
+                  new_op.id = op.id;
+                  new_op.op_type = IMarketDataListener::BookOpType::MODIFY;
+                  new_op.price = op.price;
+                  new_op.quantity = op.quantity;
+                  new_op.side = op.side;
+
+                  return update_incremental(new_op);
+               }
                if (book_side[i].price != 0 && book_side[i].price > op.price) {
                   //book_side is the book we have ?
                   //op.price is what we received ?
@@ -187,16 +199,25 @@ struct IncrementalBookData {
          switch (op.op_type) {
          case IMarketDataListener::BookOpType::Add: {
             int insert_position = 0;
-            for (int i = 0; i < fmin(book_side.size(), 5); i++) {
-               if (book_side[i].price != 0 && book_side[i].price < op.price) {
-                  insert_position++;
-                  continue;
+            for (int i = 0; i < fmin(book_side.size(),5); i++) {
+               if (book_side[i].price == op.price) {
+                  IMarketDataListener::BookOp new_op;
+                  new_op.id = op.id;
+                  new_op.op_type = IMarketDataListener::BookOpType::MODIFY;
+                  new_op.price = op.price;
+                  new_op.quantity = op.quantity;
+                  new_op.side = op.side;
+
+                  return update_incremental(new_op);
                }
-            } //need i to be at the last level
-            for (int i = fmin(book_side.size() - 1, 5 - 1); i > insert_position;
-                 i--) {
-               book_side[i].price = book_side[i - 1].price;
-               book_side[i].total_quantity = book_side[i - 1].total_quantity;
+                if (book_side[i].price != 0 && book_side[i].price < op.price) {
+                    insert_position++;
+                    continue;
+                }
+            }//need i to be at the last level
+            for (int i = fmin(book_side.size()-1,5-1); i > insert_position; i--) {
+                book_side[i].price = book_side[i - 1].price;
+                book_side[i].total_quantity = book_side[i - 1].total_quantity;
             }
             book_side[insert_position].price = op.price;
             book_side[insert_position].total_quantity = op.quantity;
@@ -290,7 +311,8 @@ struct GeneralMarketDataListener::PImpl {
    // if this is set to true, then the security key passed on in the
    // on_securities function is used. If this is not set to true, then a new key
    // is generated
-   const bool use_given_sec_key = false;
+   const bool use_given_sec_key =
+       true; // with true, replay works, with orders goign in ((did not push it in, since it was always false - need to test))
    const std::string cc_name_prefix = "cc_security";
    size_t n_bk_updates = 0;
    bts::persistence::IDGenerator &id_gen =
@@ -482,21 +504,157 @@ void GeneralMarketDataListener::on_book_increment(
    bts_assert(book_ptr != nullptr);
    _on_book(*book_ptr);
 }
+co2::AggressorSide try_fill_agg_side(
+    const domain::Price &trade_price,
+    const co2::AggressorSide &current_side,
+    const domain::Book *book,
+    const domain::SecurityID &id) {
+   if (current_side != co2::AGGRESSOR_SIDE_NONE)
+      return current_side;
+   if (!book) {
+      LOG(md, Info,
+          "No book found for sec_id: {}. Processing trade with aggressor "
+          "side set to NONE.",
+          id);
+      return current_side;
+   }
+   // Access ask-side and bid-side levels
+   auto book_ask_side = book->asks();
+   auto book_bid_side = book->bids();
+   if (book_ask_side.is_populated() &&
+       trade_price >=
+           book_ask_side[0]
+               .price) //this means the trade happened on the ask side
+   {
+      LOG(md, Info, "Trade Price: {}, Book_Ask[0] Price: {}", trade_price,
+          book_ask_side[0].price);
+      return co2::AGGRESSOR_SIDE_BUY;
+   }
+   if (book_bid_side.is_populated() &&
+       trade_price <=
+           book_bid_side[0]
+               .price) //this means the trade happened on the bid side
+   {
+      LOG(md, Info, "Trade Price: {}, Book_Ask[0] Price: {}", trade_price,
+          book_bid_side[0].price);
 
+      return co2::AGGRESSOR_SIDE_SELL;
+   }
+   std::string book_str =
+       "Bids: " + bts::io::JsonDumper::dump(book_bid_side, 2) +
+       "\nAsks: " + bts::io::JsonDumper::dump(book_ask_side, 2);
+   LOG(md, Info,
+       "Could not determine Aggressor, Trade Price: {}, Current Book: {}",
+       trade_price, book_str);
+
+   return co2::AGGRESSOR_SIDE_NONE;
+}
 void GeneralMarketDataListener::on_trades(
     const std::vector<domain::TradeRecord const *> &trades) {
    for (auto tp : trades) {
-      auto handle = _find_handle(tp->sec_id);
+
+      std::ostringstream log_stream;
+
+      // Log the important fields from the TradeRecord pointer
+      log_stream << "TradePointer - sec_id: " << tp->sec_id
+                 << ", agg_side: " << tp->agg_side
+                 << ", trade_price: " << tp->trade_price
+                 << ", trade_quantity: " << tp->trade_quantity.value()
+                 << ", time_stamp: " << tp->time_stamp.millis();
+
+      // Log the trade data
+      LOG(md, Diag, "logging tp {}", log_stream.str());
+
+      auto handle =
+          _find_handle(tp->sec_id); // might be the reason behind missing trades
+      auto aggressor_side = convert_agg_side(tp->agg_side);
+
+      // Log the initial aggressor_side
+      LOG(md, Diag, "Initial aggressor side for sec_id: {} is: {}", tp->sec_id,
+          static_cast<int>(aggressor_side));
+
+      if (handle == nullptr) {
+         LOG(md, Info, "No handle found for sec_id: {}", tp->sec_id);
+         continue;
+      }
+      aggressor_side = try_fill_agg_side(
+          tp->trade_price, aggressor_side,
+          me->inc_book_data.get_const_book(tp->sec_id), tp->sec_id);
+
       auto trade_ts = datetime::ll_from_unix_epoch_ms(tp->time_stamp.millis());
+      // Process the trade even if aggressor_side is NONE
       if (handle != nullptr) {
          handle->handle.add_trade(
              utils::TimeStamp::now(),
              PriceSize(
                  convert_price(tp->trade_price), tp->trade_quantity.value()),
-             trade_ts, convert_agg_side(tp->agg_side), 0, false);
+             trade_ts, aggressor_side, 0, false);
+         LOG(md, Diag, "Trade processed for sec_id: {}", tp->sec_id);
       }
    }
 }
+//Needed for diag for trade gaps 
+// void GeneralMarketDataListener::on_trades(
+//     const std::vector<domain::TradeRecord const *> &trades) {
+//    for (auto tp : trades) {
+
+//          for (auto tp : trades) {
+//     std::ostringstream log_stream;
+
+//     // Log the important fields from the TradeRecord pointer
+//     log_stream << "TradePointer - sec_id: " << tp->sec_id
+//                << ", agg_side: " << tp->agg_side
+//                << ", trade_price: " << tp->trade_price
+//                << ", trade_quantity: " << tp->trade_quantity.value()
+//                << ", time_stamp: " << tp->time_stamp.millis();
+
+//     // Log the trade data
+//     LOG(md, Info, "logging tp {}",log_stream.str());}
+
+//       auto handle =
+//           _find_handle(tp->sec_id); //might be the reason behind missing trades
+//       auto aggressor_side = convert_agg_side(tp->agg_side);
+//       if (aggressor_side == co2::AggressorSide::AGGRESSOR_SIDE_NONE) {
+//          auto book = me->inc_book_data.get_const_book(tp->sec_id);
+//          // Check if the book is empty
+//          if (!book) {
+//             LOG(md, Info, "book pointer log");
+//             LOG(md, Info, "Processing trade with sec_id: {}", tp->sec_id);
+//             aggressor_side =
+//                 co2::AggressorSide::AGGRESSOR_SIDE_NONE; // Reset aggressor side
+//             continue; // Skip processing this trade
+//          }else {
+//     LOG(md, Info, "Book found for sec_id: {} with size: {}", tp->sec_id, book->size());
+// }
+
+//          // Access ask-side and bid-side levels
+//          auto book_ask_side = book->asks();
+//          auto book_bid_side = book->bids();
+//           if (!book_ask_side.is_populated() || !book_bid_side.is_populated()) {
+//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_NONE; // Reset
+//               continue; // Skip processing this trade
+//           }
+//            // Determine aggressor side based on trade price
+//           if (tp->trade_price <= book_ask_side[0].price) {
+//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_BUY;
+//           } else if (tp->trade_price >= book_bid_side[0].price) {
+//               aggressor_side = co2::AggressorSide::AGGRESSOR_SIDE_SELL;
+//          }
+//       }
+
+//       //     // Check if ask and bid sides are populated
+
+//       // }
+//       auto trade_ts = datetime::ll_from_unix_epoch_ms(tp->time_stamp.millis());
+//       if (handle != nullptr) {
+//          handle->handle.add_trade(
+//              utils::TimeStamp::now(),
+//              PriceSize(
+//                  convert_price(tp->trade_price), tp->trade_quantity.value()),
+//              trade_ts, aggressor_side, 0, false);
+//       }
+//    }
+// }
 
 void GeneralMarketDataListener::on_rfqs(
     const std::vector<domain::RFQ const *> &) {}

src/lib/md/Shm/FixedSecurity.cpp

@@ -222,6 +222,10 @@ void convert_security(const FixedSecurity &source, Security *presult) {
       dest.mutable_okx()->set_security_desc(source.security_desc());
       dest.mutable_okx()->set_exchange_id(source.numeric_id);
       break;
+   case Venue::VENUE_TWSE:
+      dest.mutable_twse()->set_security_desc(source.security_desc());
+      dest.mutable_twse()->set_exchange_id(source.numeric_id);
+      break;
    case Venue::VENUE_TAIFEX:
       dest.mutable_taifex()->set_security_desc(source.security_desc());
       dest.mutable_taifex()->set_exchange_id(source.numeric_id);
@@ -334,6 +338,8 @@ uint64_t exchange_id(const Security &security) {
       return security.binance_usds_futures().exchange_id();
    case Security::ExchangeDataCase::kOkx:
       return security.okx().exchange_id();
+   case Security::ExchangeDataCase::kTwse:
+      return security.twse().exchange_id();
    case Security::ExchangeDataCase::kTaifex:
       return security.taifex().exchange_id();
    // Debatable whether this should be an error or not.
@@ -367,10 +373,15 @@ void convert_security(
       dest.security_desc(src.cboe().security_desc());
       dest.cboe_class_key = src.cboe().class_key();
       break;
+   case Security::ExchangeDataCase::kTwse:
+      dest.numeric_id = src.twse().exchange_id();
+      dest.security_desc(src.twse().security_desc());
+      // dest.cboe_class_key = src.twse().class_key();
+      break;
    case Security::ExchangeDataCase::kTaifex:
-      dest.numeric_id = src.deribit().exchange_id();
-      dest.security_desc(src.deribit().security_desc());
-      dest.cboe_class_key = src.taifex().class_key();
+      dest.numeric_id = src.taifex().exchange_id();
+      dest.security_desc(src.taifex().security_desc());
+      // dest.cboe_class_key = src.taifex().class_key();
       break;
    case Security::ExchangeDataCase::kCfe:
       dest.cboe_class_key = src.cfe().class_key();

src/lib/md/protos/co2.proto

@@ -142,6 +142,21 @@ message TradeMessage {
    AggressorSide aggressor_side = 9;
    bool from_spread = 10;
 }
+
+enum BboTradeEventType {
+   BBO_TRADE_UNKNOWN = 0;
+   //BOOK_UPDATE = 1;
+   TRADE = 2;
+}
+
+message BboTradeRecord {
+   BboTradeEventType event_type = 1;
+   // Book at time of trade 
+   TotalBook book = 2;
+   // Include trade details only for TRADE events.
+   TradeMessage trade = 3;
+}
+
 message QuoteRequestMessage {
     enum Side {
         UNDEFINED = 0;
@@ -182,6 +197,7 @@ message TotalBook{
 // Message written to the file, for replay
 message RecordedMDMessage{
    uint32 sec_key = 10;
+   string subset = 11;
    int64 timestamp = 20;
    oneof MDRecord {
       bts.api.Security security = 100;
@@ -189,6 +205,7 @@ message RecordedMDMessage{
       TradeMessage trade = 300;      
       QuoteRequestMessage rfq = 400;
       MiscMessage misc = 500;
+      BboTradeRecord bbo_trade = 600;
    }
 }
 

src/lib/md/relay/QueryServer.cpp

@@ -197,6 +197,9 @@ struct MDConsumerGroup {
       case api::VENUE_CBOE:
          this->mic = "CBOE";
          break;
+      case api::VENUE_TWSE:
+         this->mic = "TWSE";
+         break;
       case api::VENUE_TAIFEX:
          this->mic = "TAIFEX";
          break;

src/lib/orderman/gateways/OkxExecutionGateway.cpp

@@ -55,24 +55,24 @@ OkxExecutionGateway::~OkxExecutionGateway() {}
 enum class OkxExecutionState { OFF, STARTED, LOGGED_IN, SUBSCIRBED };
 void OkxExecutionGateway::send_msgs(
     std::vector<OrderMsg> const &msgs, trace::CorID) {
-   LOG(OrderMan,Info,"ExecutionGateway::send_msgs" );
+   LOG(OrderMan, Info, "ExecutionGateway::send_msgs");
    for (const auto &msg : msgs) {
       std::visit(
           [this](auto &&order) {
              using T = std::decay_t<decltype(order)>;
              if constexpr (std::is_same_v<T, CreateOrderMsg>) {
-                LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" );
+                LOG(OrderMan, Warn, "ExecutionGateway::CreateOrderMsg");
                 launch_new_order(order);
-             
+
              } else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
-                LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" );
+                LOG(OrderMan, Warn, "ExecutionGateway::CancelOrderMsg");
                 cancel_order(order);
              } else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
-                LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" );
+                LOG(OrderMan, Warn, "ExecutionGateway::CancelReplaceOrderMsg");
                 modify_order(order);
              } else if constexpr (std::is_same_v<T, RFQMsg>) {
-                 // Handle RFQMsg if necessary
-                 // adapter->handle_rfq(order); // Example placeholder
+                // Handle RFQMsg if necessary
+                // adapter->handle_rfq(order); // Example placeholder
              }
           },
           msg);
@@ -94,7 +94,8 @@ struct OkxExecutionGateway::PImpl {
    const std::string passphrase = config::get<config::OkxExeCfg>().passphrase();
    const std::string url =
        config::get<config::OkxExeCfg>().ws_url() + "/ws/v5/private";
-   const bool simulated_trading = config::get<config::OkxExeCfg>().simulated_trading();
+   const bool simulated_trading =
+       config::get<config::OkxExeCfg>().simulated_trading();
    const std::string tdmode = config::get<config::OkxExeCfg>().tdmode();
    const core::TimeSpan connect_timeout = core::TimeSpan::from_seconds(0);
    const core::TimeSpan pong_timeout = core::TimeSpan::from_seconds(
@@ -124,7 +125,8 @@ struct OkxExecutionGateway::PImpl {
       }
    }
 
-   static void dummy(std::shared_ptr<const ExecutionReport>, const trace::CorID &) {
+   static void dummy(
+       std::shared_ptr<const ExecutionReport>, const trace::CorID &) {
       LOG(OrderMan, Error, "OnixFixListener::Dummy fired");
    }
    void setCallback(const OnExeReport &callback) {
@@ -185,7 +187,7 @@ struct OkxExecutionGateway::PImpl {
           Price(std::stod(order.at("lastPx").get<std::string>()));
       report.avg_price = Price(std::stod(order.at("avgPx").get<std::string>()));
       report.last_shares = Quantity::create_new(
-        std::stod(order.at("fillSz").get<std::string>()));
+          std::stod(order.at("fillSz").get<std::string>()));
 
       //there are algos, like spreader
       // that are super sensitive to messages and cancel themselves. To avoid
@@ -195,29 +197,28 @@ struct OkxExecutionGateway::PImpl {
       // Alternatively we can change the spreader to not do that. Right not this
       // seems to be the better choice
       if (!(order.at("code").get<std::string>() == "0")) {
-        report.msg = order.at("msg").get<std::string>();
-     }
+         report.msg = order.at("msg").get<std::string>();
+      }
 
+      //report can be new order, cancel order, order filled, replace order, and partial fill or errror
+      //new order, cancelled and order filled dont need special handling, error gets handled above
+      //for other three we will look for fields that are only specific to the respectives types
+      //   if (!(order.at("cancelSource").get<std::string>() == "")) {
 
-     //report can be new order, cancel order, order filled, replace order, and partial fill or errror
-     //new order, cancelled and order filled dont need special handling, error gets handled above 
-     //for other three we will look for fields that are only specific to the respectives types
-    //   if (!(order.at("cancelSource").get<std::string>() == "")) {
-
-    //   } 
+      //   }
       if (!(order.at("amendSource").get<std::string>() == "")) {
-        if((order.at("amendResult").get<std::string>() == "0"))
+         if ((order.at("amendResult").get<std::string>() == "0"))
             report.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED;
-        else{
+         else {
             LOG(OrderMan, Warn, "order:{} replace rejected", report.order_key);
             report.msg = order.at("msg").get<std::string>();
-        }
+         }
       } else if (!(order.at("fillSz").get<std::string>() == "0")) {
-        utils::Uuid guid;
-        std::string fill_id = guid.to_text_string();
-        to_array(fill_id, &report.exchange_fill_id.value());
+         utils::Uuid guid;
+         std::string fill_id = guid.to_text_string();
+         to_array(fill_id, &report.exchange_fill_id.value());
       }
-      
+
       // report.transact_time = ExchangeTimeStampVar(std::stoll(order.at("uTime").get<std::string>()));
       ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
       report.transact_time = now;
@@ -228,14 +229,14 @@ struct OkxExecutionGateway::PImpl {
       bts_assert_msgf(
           false, "Websocket stream client disconnected. {}", ec.message());
    }
-    // fills execution report form basic_json and writes it to
-    // buffered_exec_reps, which is picked up from the other thread
-    // and passed over to the order infrastructure
-    void _push_execution_report(const nlohmann::basic_json<> &order) {
-        ExecutionReport er;
-        get_exe_report(order, er);
-        buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
-    }
+   // fills execution report form basic_json and writes it to
+   // buffered_exec_reps, which is picked up from the other thread
+   // and passed over to the order infrastructure
+   void _push_execution_report(const nlohmann::basic_json<> &order) {
+      ExecutionReport er;
+      get_exe_report(order, er);
+      buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
+   }
    void on_message(const std::string &msg_str) {
       LOG(OrderMan, Info, "okxlistener::on_message. {}", msg_str);
       nlohmann::json msg;
@@ -279,21 +280,22 @@ struct OkxExecutionGateway::PImpl {
       }
 
       const auto &data = msg.at("data");
-      
-
 
       for (const nlohmann::basic_json<> &order : data) {
-        int orderkey = 0;
-        try {
+         int orderkey = 0;
+         try {
             int temp = std::stoi(order.at("clOrdId").get<std::string>());
             orderkey = temp;
-        }catch (...) {
-        }
-        if(orderkey ==0){
-            LOG(OrderMan, Error, "okxlistener::on_message GOT ER FOR ORDER NOT ENTERED FROM SYSTEM. {}", msg_str);
+         } catch (...) {
+         }
+         if (orderkey == 0) {
+            LOG(OrderMan, Error,
+                "okxlistener::on_message GOT ER FOR ORDER NOT ENTERED FROM "
+                "SYSTEM. {}",
+                msg_str);
             continue;
-        }
-        _push_execution_report(order);
+         }
+         _push_execution_report(order);
       }
    }
    void start() {
@@ -456,8 +458,8 @@ struct OkxExecutionGateway::PImpl {
 };
 void OkxExecutionGateway::start() { me_->start(); }
 OnGwStatusChange &OkxExecutionGateway::on_gw_status_change() {
-    return me_->on_gw_status_change;
- }
+   return me_->on_gw_status_change;
+}
 void OkxExecutionGateway::modify_order(
     const CancelReplaceOrderMsg &msg) { //confirm will have everything
    try {
@@ -474,7 +476,7 @@ void OkxExecutionGateway::modify_order(
           std::to_string(msg.create_fields.quantity.full_value()));
       auto headers = OkxUtils::createHeaders(
           me_->api_key, me_->passphrase, me_->secret_key, "POST", REQUEST_PATH,
-          me_->simulated_trading,body);
+          me_->simulated_trading, body);
       // Generate and log the curl command
       std::string curlCommand = OkxUtils::generateCurlCommand(
           config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, headers,
@@ -488,7 +490,8 @@ void OkxExecutionGateway::modify_order(
       if (suc) {
          LOG(md, Info, "Http get succeeded {}", res.body);
       } else {
-        LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}", res.code, res.errors, res.body);
+         LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}",
+             res.code, res.errors, res.body);
       }
    } catch (const std::exception &ex) {
       const std::string err = ex.what();
@@ -496,17 +499,17 @@ void OkxExecutionGateway::modify_order(
    }
 }
 void OkxExecutionGateway::dispatch_incoming_fix_messages() {
-    if (me_->buffered_exec_reps.empty()) {
-        return;
-     }
-     std::shared_ptr<ExecutionReport> er;
-     while (me_->buffered_exec_reps.try_pop(er)) {
-        trace::CorID cid = {};
-        LOG(OrderMan, Info,
-            "OkxExecutionGateway::dispatch_incoming_fix_messages er: {} ",
-            er->to_string());
-        me_->cb(std::move(er), cid);
-     }
+   if (me_->buffered_exec_reps.empty()) {
+      return;
+   }
+   std::shared_ptr<ExecutionReport> er;
+   while (me_->buffered_exec_reps.try_pop(er)) {
+      trace::CorID cid = {};
+      LOG(OrderMan, Info,
+          "OkxExecutionGateway::dispatch_incoming_fix_messages er: {} ",
+          er->to_string());
+      me_->cb(std::move(er), cid);
+   }
 }
 OnExeReport &OkxExecutionGateway::on_exe_report() { return me_->cb; }
 OnCancelReject &OkxExecutionGateway::on_cancel_reject() {
@@ -535,15 +538,16 @@ void OkxExecutionGateway::launch_new_order(const CreateOrderMsg &msg) {
       std::string suffix = "ORDER_TYPE_";
       auto ord_full_name = NativeOrderType_Name(msg.order_type);
       auto pos = ord_full_name.find(suffix);
-      const std::string ord_type =
+      const std::string ord_type =msg.time_in_force==api::orderman::TIME_IN_FORCE_IOC?"ioc":
           FIX::string_toLower(ord_full_name.substr(pos + suffix.size()));
+    
       const std::string body = fmt::format(
-          body_template, msg.security->okx().security_desc(),me_->tdmode, side, ord_type,
-          price, quantity, msg.key);
+          body_template, msg.security->okx().security_desc(), me_->tdmode, side,
+          ord_type, price, quantity, msg.key);
       http::HttpResponse res;
       auto headers = OkxUtils::createHeaders(
           me_->api_key, me_->passphrase, me_->secret_key, "POST", REQUEST_PATH,
-          me_->simulated_trading,body);
+          me_->simulated_trading, body);
       // Generate and log the curl command
       std::string curlCommand = OkxUtils::generateCurlCommand(
           config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, headers,
@@ -556,30 +560,33 @@ void OkxExecutionGateway::launch_new_order(const CreateOrderMsg &msg) {
       const auto &[suc, top_msg] = OkxUtils::orderAccepted(res.body);
 
       if (suc) {
-        LOG(md, Info, "Http get succeeded {}", res.body);
-       //  er.exec_type = api::orderman::ExecType::EXEC_TYPE_PENDING_NEW;
-       //  er.order_status = api::orderman::NATIVE_ORDER_STATUS_PENDING_NEW;
-     } else {
-        LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}", res.code, res.errors, res.body);
-        OkxUtils::populateCreateOrderFields(er, msg);
-       er.price = msg.price;
-       er.quantity = msg.quantity;
-       er.filled_quantity = 0;
-       er.last_shares = 0;
-       er.leg_security = nullptr;
-       er.avg_price = Price(0);
-       ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
-       er.transact_time = now;
-       er.order_key = msg.key;
-       const auto &[id, ts, ex_msg] = OkxUtils::extractOrderInfo(res.body);
-       er.exchange_order_id = ExchangeOrderID{OkxUtils::stringToFixedArray(id)};
-       er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
-       er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
-       er.msg = ex_msg + " " + top_msg;
-       me_->buffered_exec_reps.push(
-           std::make_shared<ExecutionReport>(std::move(er)));
+         LOG(md, Info, "Http get succeeded {}", res.body);
+         //  er.exec_type = api::orderman::ExecType::EXEC_TYPE_PENDING_NEW;
+         //  er.order_status = api::orderman::NATIVE_ORDER_STATUS_PENDING_NEW;
+      } else {
+         LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}",
+             res.code, res.errors, res.body);
+         OkxUtils::populateCreateOrderFields(er, msg);
+         er.price = msg.price;
+         er.quantity = msg.quantity;
+         er.filled_quantity = 0;
+         er.last_shares = 0;
+         er.leg_security = nullptr;
+         er.avg_price = Price(0);
+         ExchangeTimeStamp64 now =
+             ExchangeTimeStamp64(datetime::unix_nano_now());
+         er.transact_time = now;
+         er.order_key = msg.key;
+         const auto &[id, ts, ex_msg] = OkxUtils::extractOrderInfo(res.body);
+         er.exchange_order_id =
+             ExchangeOrderID{OkxUtils::stringToFixedArray(id)};
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+         er.msg = ex_msg + " " + top_msg;
+         me_->buffered_exec_reps.push(
+             std::make_shared<ExecutionReport>(std::move(er)));
+      }
    }
-  }
    // Shut down all reactors if they exist.
    catch (const std::exception &ex) {
       const std::string err = ex.what();
@@ -605,7 +612,7 @@ void OkxExecutionGateway::cancel_order(const CancelOrderMsg &msg) {
           body_template, msg.security->okx().security_desc(), msg.key);
       auto headers = OkxUtils::createHeaders(
           me_->api_key, me_->passphrase, me_->secret_key, "POST", REQUEST_PATH,
-          me_->simulated_trading,body);
+          me_->simulated_trading, body);
       // Generate and log the curl command
       std::string curlCommand = OkxUtils::generateCurlCommand(
           config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, headers,
@@ -630,7 +637,8 @@ void OkxExecutionGateway::cancel_order(const CancelOrderMsg &msg) {
          //  er.exec_type = api::orderman::ExecType::EXEC_TYPE_PENDING_CANCEL;
          //  er.order_status = api::orderman::NATIVE_ORDER_STATUS_PENDING_CANCEL;
       } else {
-        LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}", res.code, res.errors, res.body);
+         LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}",
+             res.code, res.errors, res.body);
          //  er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
          //  er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
          //  er.msg = ex_msg + " " + top_msg;
@@ -643,8 +651,7 @@ void OkxExecutionGateway::cancel_order(const CancelOrderMsg &msg) {
       OkxUtils::handleOkException("Cacnel order", err);
    }
 }
-} // namespace bts::orderman::okxexe
-
+} // namespace bts::orderman
 
 // ExecutionReport OkxExecutionAdapter::fire_dummy_modify_order_report(CancelReplaceOrderMsg msg) {
 //     ExecutionReport *er_p = me_->create_exe_ptr();

src/lib/orderman/gateways/sim/SimExecutionGateway.cpp

@@ -14,6 +14,8 @@
 #include "lib/md/MDProducer.h"
 #include "lib/md/PriceSize.h"
 #include "lib/md/Shm/MDStructure.h"
+#include "lib/orderman/CancelOrderMsg.h"
+#include "lib/orderman/CreateOrderMsg.h"
 #include "lib/orderman/OrderMsg.h"
 #include "lib/orderman/events/NewAdminDataEvent.h"
 #include "lib/orderman/events/NewSecuritiesEvent.h"
@@ -180,6 +182,10 @@ struct SEGOrderState {
       last_filled_qty = 0;
       last_filled_price = Price(os::nan());
    }
+   bool is_ioc() const {
+      return crt_msg.time_in_force ==
+             api::orderman::TimeInForce::TIME_IN_FORCE_IOC;
+   }
 };
 
 struct SEGTradeEntry {
@@ -226,8 +232,10 @@ struct SEGSecurityData {
          const CancelReplaceOrderMsg &cxlr_msg =
              std::get<CancelReplaceOrderMsg>(msg);
          SEGOrderState *seg_os = find_order(cxlr_msg.create_fields.key);
-         if(seg_os == nullptr){
-            LOG(OrderMan, Info, "SimExecutionGateway::apply_order_action tried updating unkown order");
+         if (seg_os == nullptr) {
+            LOG(OrderMan, Info,
+                "SimExecutionGateway::apply_order_action tried updating unkown "
+                "order");
             return {ActionResult::NoOp, nullptr};
          }
          auto action_result = seg_os->apply_change(cxlr_msg);
@@ -235,8 +243,10 @@ struct SEGSecurityData {
       } else if (std::holds_alternative<CancelOrderMsg>(msg)) {
          const CancelOrderMsg &cxl_msg = std::get<CancelOrderMsg>(msg);
          SEGOrderState *seg_os = find_order(cxl_msg.key);
-         if(seg_os == nullptr){
-            LOG(OrderMan, Info, "SimExecutionGateway::apply_order_action tried canceling unkown order");
+         if (seg_os == nullptr) {
+            LOG(OrderMan, Info,
+                "SimExecutionGateway::apply_order_action tried canceling "
+                "unkown order");
             return {ActionResult::NoOp, nullptr};
          }
          auto action_result = seg_os->apply_cxl(cxl_msg);
@@ -578,6 +588,14 @@ bts::orderman::Book SimExecutionGateway::_match_order(
    if (crosses_opp_side) {
       return _cross_order(book, seg_os);
    } else {
+      if (seg_os.is_ioc()) {
+         CancelOrderMsg cxl_msg;
+         cxl_msg.key = seg_os.ord_key();
+         seg_os.apply_cxl(cxl_msg);
+         _push_execution_report(seg_os);
+         // generate cancel execution report
+         return book;
+      }
       return _work_order(book, seg_os);
    }
 }
@@ -651,7 +669,8 @@ SEGSecurityData *SimExecutionGateway::_find_add_security_entry(
 void SimExecutionGateway::_push_execution_report(const SEGOrderState &seg_os) {
    ExecutionReport er;
    fill_execution_report(seg_os, er);
-   me->buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
+   me->buffered_exec_reps.push(
+       std::make_shared<ExecutionReport>(std::move(er)));
 }
 
 void SimExecutionGateway::_write_book(SecurityKey key) {

src/lib/orderman/orders/spreader_orders/SpreaderOrder.cpp

@@ -156,12 +156,15 @@ void SpreaderOrder::_on_time(const bts::mercury::TimerEvent &t_event) {
    }
 }
 void SpreaderOrder::_on_book_update(const BookUpdateEvent &book_event) {
-   // TODO: rmeove
    LOG(OrderMan, Diag,
        "SpreaderOrder::_on_book_update skey: {} bid: {}/{} a: {}/{}",
        book_event.key(), book_event.book.bid().at(0).price(),
        book_event.book.bid().at(0).size(), book_event.book.ask().at(0).price(),
        book_event.book.ask().at(0).size());
+   if (book_event.key() == _bookkeeping_data().spread_sec_key()) {
+      _bookkeeping_data().spread_book = book_event.book;
+      return;
+   }
    if (!_active)
       return;
    auto scoped = _order_services->log_scope(

src/lib/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -18,6 +18,7 @@
 #include "lib/orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.h"
 #include "lib/orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.h"
 #include "lib/sparkproto/SparkProtos.pb.h"
+#include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/is_unittest_environment.h"
 #include <memory>
@@ -107,7 +108,7 @@ class SpreaderOrder : public Order, public ISpreaderManager {
  public:
    // inherited function to return the OrderDefinition. asserts
    // if on_init_new has not been called yet
-   const api::orderman::OrderDefinition &definition() const;
+   const api::orderman::OrderDefinition &definition() const override;
    // this is the construction, can be slow
    void on_init_new(
        const bts::api::orderman::OrderDefinition &order_def,
@@ -136,8 +137,19 @@ class SpreaderOrder : public Order, public ISpreaderManager {
    bts::orderman::OrderKey spreader_order_key() const override { return key(); }
    bts::TimeStamp current_time() const override { return _curr_timestamp; }
    const bts::orderman::Book &get_book(size_t leg_index) const override {
-      return _bookkeeping_data().get_leg_data(leg_index).book();
+      const SpreaderLegData &leg_data =
+          _bookkeeping_data().get_leg_data(leg_index);
+      const auto &rv = leg_data.book();
+      return rv;
+      //return _bookkeeping_data().get_leg_data(leg_index).book();
    }
+
+   const bts::orderman::Book &get_spread_book() const override {
+      const auto &opt_book = _bookkeeping_data().spread_book;
+      bts_assert(opt_book.has_value());
+      return opt_book.value();
+   }
+
    const LegPropertyData &get_leg_data(size_t leg_index) const override {
       return _bookkeeping_data().get_leg_data(leg_index).leg_property_data();
    }
@@ -148,6 +160,10 @@ class SpreaderOrder : public Order, public ISpreaderManager {
    inline bool in_cache_warming_cycle() const override {
       return _cw_helper.in_cache_warming_cycle;
    }
+   inline bts::spark::ExecutionStrategy::SpreaderType
+   get_spreader_quoting_type() const override {
+      return _bookkeeping_data().spreader_quoting_type();
+   }
    // this is called by the SpreaderChildOrderState
    void place_order(
        ChildOrder &child_order,

src/lib/orderman/orders/spreader_orders/spreader/ISpreaderManager.h

@@ -1,9 +1,12 @@
 #pragma once
 #include "SpreaderFractionalQtyHelper.h"
+#include "lib/api/services/OrderService.pb.h"
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/domain_model/common.h"
 #include "lib/orderman/ChildOrder.h"
 #include "lib/orderman/orderman_utils/Book.h"
+#include "lib/sparkproto/SparkProtos.pb.h"
+#include <optional>
 
 namespace bts {
 namespace orderman {
@@ -18,10 +21,17 @@ class IAlternateOrderGateway;
 // ii. create new quoter order
 class ISpreaderManager {
  public:
+   // definition() was added when we expanded the spreader to add reverse hedging and sniper.
+   // that needed spreader sid etc, which is all in the definition. Also spreader_order_key() can
+   // retired
+   virtual const api::orderman::OrderDefinition &definition() const = 0;
    virtual bts::orderman::OrderKey spreader_order_key() const = 0;
    virtual bts::TimeStamp current_time() const = 0;
    // this is a reference to the original book structure
    virtual const bts::orderman::Book &get_book(size_t leg_index) const = 0;
+   // it returns the valid spread book, for quoting tsrategies (non re-quoting) that need the
+   // the spread book. otherwise this function asserts
+   virtual const bts::orderman::Book &get_spread_book() const = 0;
    // the caller can keep a reference to the returned type forever. the
    // data in the returned type is all const
    virtual const LegPropertyData &get_leg_data(size_t leg_index) const = 0;
@@ -41,6 +51,8 @@ class ISpreaderManager {
    // (however do as much as they can to keep the instruction
    // cache warm)
    virtual bool in_cache_warming_cycle() const = 0;
+   virtual bts::spark::ExecutionStrategy::SpreaderType
+   get_spreader_quoting_type() const = 0;
    // orders are placed through the ISpreaderManager. Eventhough ChildOrder has
    // the facility to do that directly, we want to control all order activity
    // through one central place

src/lib/orderman/orders/spreader_orders/spreader/LegPropertyData.cpp

@@ -1,6 +1,8 @@
 #include "LegPropertyData.h"
 #include "ISpreaderManager.h"
 #include "SpreaderUtils.h"
+#include "lib/sparkproto/SparkProtos.pb.h"
+#include "lib/utils/bts_assert.h"
 
 namespace bts {
 namespace orderman {
@@ -10,11 +12,37 @@ LegPropertyData::LegPropertyData(
     const bts::api::ContractLeg &leg_contract,
     const bts::spark::LegSpreaderParams &leg_params,
     api::Side trading_side,
+    bts::spark::ExecutionStrategy::SpreaderType spreader_quoting_type,
     bts::api::SyntheticContract::PriceCalc price_calc,
     double qty_tick_size)
     : _leg_index(leg_index), _leg_contract(leg_contract),
       _leg_params(leg_params), _trading_side(trading_side),
+
       _price_calc(price_calc), _qty_tick_size(qty_tick_size) {
+   switch (spreader_quoting_type) {
+   case bts::spark::ExecutionStrategy::SpreaderType::
+       ExecutionStrategy_SpreaderType_REQUOTE: {
+      // for historic reasons, we use the boolean only in the requote case
+      _is_active_quoting = //leg_params.has_quote_params();
+          leg_params.is_active_quoting();
+      break;
+   }
+   case bts::spark::ExecutionStrategy::SpreaderType::
+       ExecutionStrategy_SpreaderType_REVERSE_HEDGE: {
+      _is_active_quoting = leg_params.has_reverse_hedge_quote_params();
+      break;
+   }
+   case bts::spark::ExecutionStrategy::SpreaderType::
+       ExecutionStrategy_SpreaderType_SNIPER: {
+      _is_active_quoting = leg_params.has_sniper_quote_params();
+      break;
+   }
+   default: {
+      bts_assert(false);
+      break;
+   }
+   }
+
    LOG(OrderMan, Info,
        "LegPropertyData.cons leg_idx: "
        "{} sec_key: {} side: {}",

src/lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h

@@ -19,6 +19,7 @@ class LegPropertyData {
    const api::Side _trading_side;
    const bts::api::SyntheticContract::PriceCalc _price_calc;
    const double _qty_tick_size = 1.0;
+   bool _is_active_quoting;
 
  public:
    LegPropertyData(
@@ -26,6 +27,9 @@ class LegPropertyData {
        const bts::api::ContractLeg &leg_contract,
        const bts::spark::LegSpreaderParams &leg_params,
        api::Side trading_side,
+       bts::spark::ExecutionStrategy::SpreaderType spreader_quoting_type =
+           bts::spark::ExecutionStrategy::SpreaderType::
+               ExecutionStrategy_SpreaderType_REQUOTE,
        bts::api::SyntheticContract::PriceCalc price_calc =
            bts::api::SyntheticContract::PRICECALC_STANDARD,
        double qty_tick_size = 1.0);
@@ -50,7 +54,9 @@ class LegPropertyData {
       return _leg_contract.security().contract_spec().low_tick();
    }
    double qty_tick_size() const { return _qty_tick_size; }
-   bool is_active_quoting() const { return _leg_params.is_active_quoting(); }
+   // TODO: is_quoting
+   //bool is_active_quoting() const { return _leg_params.is_active_quoting(); }
+   bool is_active_quoting() const { return _is_active_quoting; }
    bool is_net_change() const {
       return _price_calc == bts::api::SyntheticContract::PRICECALC_NET_CHANGE;
    }

src/lib/orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp

@@ -76,8 +76,7 @@ SingleLegQuoter::SingleLegQuoter(SingleLegQuoter &&other)
    bts_assert(false);
 }
 /*********************************************************************************/
-/*********************************spreader
- * calls**********************************/
+/*********************************spreader calls**********************************/
 /*********************************************************************************/
 void SingleLegQuoter::one_time_activate(Price price, Quantity quantity) {
    bts_assert(_active_quoting);
@@ -274,7 +273,9 @@ void SingleLegQuoter::_transform_to_WAITING_FOR_IOC() {
    _switch_state(&_waiting_for_ioc_worker);
 }
 void SingleLegQuoter::_transform_to_WAITING_FOR_FIRST_ACK_CANCELLED() {
-   bts_assert(_worker->state() == QuoterState::WAITING_FOR_FIRST_ACK);
+   bts_assert(
+       _worker->state() == QuoterState::WAITING_FOR_FIRST_ACK ||
+       _worker->state() == QuoterState::WAITING_FOR_IOC);
    _switch_state(&_waiting_for_first_ack_cancelled_state_worker);
 }
 void SingleLegQuoter::_transform_to_WORKING() {

src/lib/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp

@@ -4,6 +4,7 @@
 #include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.h"
 #include "lib/os/os.h"
+#include "lib/utils/PBUtils.h"
 
 namespace bts {
 namespace orderman {
@@ -41,6 +42,11 @@ SpreaderBookkeepingData::SpreaderBookkeepingData(
       _leg_data.emplace_back(
           spreader_pao_services, spreader_order_key, _n_legs, spreader_manager,
           quoter_ord_defs[i], hedger_ord_defs[i], leg_property_data);
+      LOG(OrderMan, Info,
+          "SpreaderBookkeepingData::cons leg_idx: {} quoter_ord_def: {} "
+          "hedger_ord_defs: {}",
+          i, utils::pb::to_string(quoter_ord_defs[i]),
+          utils::pb::to_string(hedger_ord_defs[i]));
       bts_assert(
           _sec_key_to_index.find(leg_property_data.security_key()) ==
           _sec_key_to_index.end());

src/lib/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.h

@@ -1,8 +1,11 @@
 #pragma once
+#include "lib/domain_model/SecurityKey.h"
 #include "lib/logging/logging.h"
+#include "lib/orderman/orderman_utils/Book.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include <boost/container/flat_map.hpp>
 #include <memory>
+#include <optional>
 
 #include "NLegsHedgeRecompData.h"
 #include "SpreaderLegData.h"
@@ -14,6 +17,7 @@
 #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.h"
 #include "lib/orderman/orders/spreader_orders/spreader/hedger/HedgerManager.h"
 #include "lib/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h"
+#include "lib/sparkproto/SparkProtos.pb.h"
 
 namespace bts {
 namespace orderman {
@@ -42,6 +46,9 @@ class SpreaderBookkeepingData {
    NLegsHedgeRecompData _hedge_recomp_data;
    SpreaderFractionalQtyHelper _fqty_helper;
 
+ public:
+   std::optional<Book> spread_book;
+
  private:
    size_t _calc_n_quoting_legs(const std::vector<LegPropertyData> &legs);
 
@@ -166,6 +173,15 @@ class SpreaderBookkeepingData {
    }
    size_t n_quoting_legs() const { return _n_quoting_legs; }
    SpreaderFractionalQtyHelper &fqty_helper() { return _fqty_helper; }
+   SecurityKey spread_sec_key() const {
+      return SecurityKey(_order_def.security_key());
+   }
+   bts::spark::ExecutionStrategy::SpreaderType spreader_quoting_type() const {
+      return _order_def.custom_def()
+          .spreader()
+          .execution_strategy()
+          .spreader_type();
+   }
 };
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/spreader_orders/spreader/SpreaderLegData.h

@@ -96,7 +96,10 @@ class SpreaderLegData {
       bts_assert(_is_quoting);
       return *_quoter;
    }
-   const bts::orderman::Book &book() const { return _book; }
+   const bts::orderman::Book &book() const {
+      // book
+      return _book;
+   }
    Price settlement_price() const { return Price(_settlement_price.price()); }
    bool settlement_populated() const {
       return !os::is_nan(_settlement_price.price());
@@ -162,6 +165,11 @@ class SpreaderLegData {
           book, _settlement_price, _leg_property_data.tick_size(),
           _leg_property_data.md_type(),
           bts::api::SyntheticContract::PRICECALC_STANDARD, _book);
+      // LOG(OrderMan, Info,
+      //     "SpreaderLegData::update_book index: {} lpd_key: {} book: {} book "
+      //     "key: {}",
+      //     _leg_property_data.leg_index(), _leg_property_data.security_key(),
+      //     book, book.security_key());
    }
    void update_settlement_price(md::IntPrice price) {
       _settlement_price = price;

src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp

@@ -1,11 +1,15 @@
 #include "SpreaderUtils.h"
 #include "lib/domain_model/common.h"
+#include "lib/logging/logging.h"
 #include "lib/md/PriceSize.h"
 #include "lib/orderman/events/BookUpdateEvent.h"
 #include "lib/orderman/orderman_utils/Book.h"
 #include "lib/orderman/orders/PreActivationOrderServices.h"
 #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.h"
 #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.h"
+#include "lib/sparkproto/SparkProtos.pb.h"
+#include "lib/utils/PBUtils.h"
+#include "lib/utils/bts_assert.h"
 #include <sstream>
 namespace bts {
 namespace orderman {
@@ -65,6 +69,28 @@ bool SpreaderUtils::resolve_synthetic_spread(
     std::vector<bts::api::orderman::OrderDefinition> &out_quoter_defs,
     std::vector<bts::api::orderman::OrderDefinition> &out_hedger_defs,
     bool resolve_synthetic) {
+   LOG(OrderMan, Info, "SpreaderUtils::resolve_synthetic_spread order_def: {}",
+       utils::pb::to_string(order_def));
+   auto get_quote_time_enforce =
+       [&](const spark::ExecutionStrategy &exec_strat,
+           const spark::LegSpreaderParams &q_leg_params) {
+          switch (exec_strat.spreader_type()) {
+          case spark::ExecutionStrategy::SpreaderType::
+              ExecutionStrategy_SpreaderType_REQUOTE:
+          case spark::ExecutionStrategy::SpreaderType::
+              ExecutionStrategy_SpreaderType_SNIPER:
+             return order_def.time_in_force();
+          case spark::ExecutionStrategy::SpreaderType::
+              ExecutionStrategy_SpreaderType_REVERSE_HEDGE:
+             if (q_leg_params.has_reverse_hedge_quote_params())
+                return bts::api::orderman::TimeInForce::TIME_IN_FORCE_IOC;
+             else
+                return order_def.time_in_force();
+          default:
+             bts_assert(false);
+             return order_def.time_in_force();
+          }
+       };
    bts_assert(
        order_def.has_custom_def() && order_def.custom_def().has_spreader());
    const bts::api::Security *sec_ptr =
@@ -128,27 +154,32 @@ bool SpreaderUtils::resolve_synthetic_spread(
                          ->mutable_limit()
                          ->MergeFrom(api::orderman::LimitOrderDefinition());
                   }
-
-                  if (leg_params.quote_params().has_quote_hidden())
-                     quoter_order_def.set_time_in_force(
-                         bts::api::orderman::TimeInForce::TIME_IN_FORCE_IOC);
-                  else
-                     quoter_order_def.set_time_in_force(
-                         order_def.time_in_force());
+                  auto tif =
+                      get_quote_time_enforce(execution_strategy, leg_params);
+                  quoter_order_def.set_time_in_force(tif);
                   // hedger order def
                   api::orderman::OrderDefinition hedger_order_def =
                       quoter_order_def;
                   hedger_order_def.mutable_custom_def()
                       ->mutable_limit()
                       ->MergeFrom(api::orderman::LimitOrderDefinition());
-                  quoter_order_def.set_time_in_force(order_def.time_in_force());
+                  //quoter_order_def.set_time_in_force(order_def.time_in_force());
                   hedger_order_def.set_sub_type(
                       bts::api::orderman::OrderSubType::
                           ORDER_SUB_TYPE_HEDGING_LEG);
                   LegPropertyData leg_property_data(
                       leg_idx, leg, leg_params, leg_order_side,
+                      order_def.custom_def()
+                          .spreader()
+                          .execution_strategy()
+                          .spreader_type(),
                       sec.synthetic().price_calc());
                   out_resolved.push_back(leg_property_data);
+                  LOG(OrderMan, Info,
+                      "SpreaderUtils::resolve_synthetic_spread idx: {} "
+                      "quoter_order_def: {}",
+                      out_quoter_defs.size(),
+                      utils::pb::to_string(quoter_order_def));
                   out_quoter_defs.push_back(quoter_order_def);
                   out_hedger_defs.push_back(hedger_order_def);
                   leg_idx++;

src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h

@@ -5,6 +5,7 @@
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/common.h"
 #include "lib/jetfix/codec/FieldDefinitions.h"
+#include "lib/logging/logging.h"
 #include "lib/md/IntPrice.h"
 #include "lib/md/PriceSize.h"
 #include "lib/md/Shm/MDStructure.h"
@@ -20,6 +21,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <random>
+#include <tuple>
 #include <utility>
 
 namespace bts {
@@ -312,8 +314,10 @@ class SpreaderUtils {
                   opp_side[0].price(), price.value(), tick_size));
    }
    // find the top of the synthetic book
-   static Price top_of_book_price(const ISpreaderManager &, api::Side) {
-      Price rv(os::nan());
+   static std::tuple<Price, Quantity> top_of_book_price(
+       const ISpreaderManager &, api::Side) {
+      //TODO
+      std::tuple<Price, Quantity> rv{os::nan(), Quantity::create_new(0.0)};
       return rv;
    }
 
@@ -459,6 +463,75 @@ class SpreaderUtils {
       }
    }
 
+   // this is a general purpose function that tells if the book is through the
+   // order price by a certian number of ticks
+   static int book_through_order_price_nticks(
+       Price order_price,
+       api::Side order_side,
+       double tick_size,
+       const Book &book) {
+      // LOG(OrderMan, Info,
+      //     "SpreaderUtils::book_through_order_price_nticks: order_price: {} "
+      //     "order_side: {} tick_size: {} -- bid: {}/{} ask: {}/{}",
+      //     order_price, static_cast<int>(order_side), tick_size,
+      //     book.bid().at(0).price(), book.bid().at(0).size(),
+      //     book.ask().at(0).price(), book.ask().at(0).size());
+      if (order_side == api::SIDE_BUY) {
+         const md::PriceSize &top_ask = book.ask()[0];
+         if (top_ask.size() == 0)
+            return -100;
+         int t_diff = securities::price_diff_to_ticks(
+             top_ask.price(), order_price.value(), tick_size);
+         return t_diff;
+      } else {
+         const md::PriceSize &top_bid = book.bid()[0];
+         if (top_bid.size() == 0)
+            return -100;
+         int t_diff = securities::price_diff_to_ticks(
+             order_price.value(), top_bid.price(), tick_size);
+         return t_diff;
+      }
+   }
+
+   static Price compute_order_price_through_book(
+       const Book &book, api::Side order_side, double tick_size, int n_ticks) {
+      // LOG(OrderMan, Info,
+      //     "SpreaderUtils::compute_order_price_through_book tick_size: {} "
+      //     "n_ticks: {}",
+      //     tick_size, n_ticks);
+      switch (order_side) {
+      case api::Side::SIDE_BUY: {
+         if (n_ticks >= 0) {
+            double top_ask = book.ask()[0].price();
+            return Price(top_ask + tick_size * n_ticks);
+            //return Price(book.ask()[n_ticks].price());
+         } else {
+            double top_bid = book.bid()[0].price();
+            return Price(top_bid - tick_size * n_ticks);
+            //return Price(book.bid()[std::abs(n_ticks)].price());
+         }
+      }
+      case api::Side::SIDE_SELL: {
+         if (n_ticks >= 0) {
+            double top_bid = book.bid()[0].price();
+            double rv = top_bid - tick_size * n_ticks;
+            LOG(OrderMan, Info,
+                "SpreaderUtils::compute_order_price_through_book top_bid: {} "
+                "rv: {}",
+                top_bid, rv);
+            return Price(rv);
+         } else {
+            double top_ask = book.ask()[0].price();
+            return Price(top_ask + tick_size * n_ticks);
+         }
+      }
+      default: {
+         bts_assert(false);
+         return Price(os::nan());
+      }
+      }
+   }
+
    ////////////////////////////////////////////////////
    // pre-hedge params ////////////////////////////////
    ////////////////////////////////////////////////////

src/lib/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp

@@ -1,8 +1,18 @@
 #include "lib/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
+#include "lib/logging/logging.h"
+#include "lib/md/PriceSize.h"
+#include "lib/orderman/orderman_utils/Book.h"
 #include "lib/orderman/orders/spreader_orders/spreader/ISpreaderManager.h"
+#include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SingleLegQuoter.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.h"
+#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
 #include "lib/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterState.h"
+#include "lib/os/os.h"
+#include "lib/sparkproto/SparkProtos.pb.h"
+#include "lib/utils/bts_assert.h"
 
 namespace bts {
 namespace orderman {
@@ -53,12 +63,112 @@ void QuoterWaitingForSizeStateWorker::_longpath_analyze_and_act() {
           "target reached, transforming to end",
           _quoter->_key().value());
       _quoter->_transform_to_END();
+      return;
    } else {
+      auto spreader_q_type = _quoter->_spreader.get_spreader_quoting_type();
+      switch (spreader_q_type) {
+      case bts::spark::ExecutionStrategy::SpreaderType::
+          ExecutionStrategy_SpreaderType_REQUOTE: {
+         Price out_price;
+         Quantity out_quantity;
+         _quoter->_compute_price_quantity(out_price, out_quantity);
+         if (out_quantity.value() > 0) {
+            // only applicable for > 2 legged spreads
+            _quoter->_commit_drivers(out_price);
+            LOG(OrderMan, Info,
+                "QuoterWaitingForSizeStateWorker (requoting) sending order "
+                "key:{} price:{} quantity:{}",
+                _quoter->_key(), out_price, out_quantity);
+            // this is the only place where a quoter order is entered
+            _quoter->_place_order(out_price, out_quantity);
+            _quoter->_transform_to_WAITING_FOR_FIRST_ACK();
+         }
+         return;
+      }
+      case bts::spark::ExecutionStrategy::SpreaderType::
+          ExecutionStrategy_SpreaderType_REVERSE_HEDGE: {
+         auto is_book_populated = [&](const Book &leg_book, api::Side bs) {
+            const auto &book_side = leg_book.side(bs);
+            return !os::is_nan(book_side.at(0).price());
+         };
+         const LegPropertyData &leg_data = _quoter->_this_leg_data;
+         bts_assert(leg_data.leg_params().has_reverse_hedge_quote_params());
+         const auto &rev_hedge_qparams =
+             leg_data.leg_params().reverse_hedge_quote_params();
+         int n_ticks = rev_hedge_qparams.n_ticks();
+         const Book &spread_book = _quoter->_spreader.get_spread_book();
+         auto spread_side = _quoter->_spreader.definition().side();
+         double spread_book_tick_sz = _quoter->_spreader.definition()
+                                       .custom_def()
+                                       .spreader()
+                                       .security()
+                                       .contract_spec()
+                                       .low_tick();
+
+         int book_through_ticks =
+             SpreaderUtils::book_through_order_price_nticks(
+                 _quoter->_price_quantity_state.target_spread_price(),
+                 // TODO (get it from the spread synth sec)
+                 spread_side, spread_book_tick_sz, spread_book);
+         LOG(OrderMan, Info,
+             "QuoterWaitingForSizeStateWorker::_longpath_analyze_and_act("
+             "reverse hedge) spread_book_tick_sz: {}"
+             "n_ticks: {} book_through_ticks: {}",
+             spread_book_tick_sz, n_ticks, book_through_ticks);
+         if (book_through_ticks >= n_ticks) {
+            // qty, shoot full size? (can min it by max available on hedge leg)
+            //auto spr_qty = _quoter->_price_quantity_state.out_quantity();
+            // needs to be reduced by filled qty
+            // qty: (this is what remains, reduced by how can be leaned on)
+            auto order_qty = _quoter->_price_quantity_state.remaining_to_fill();
+            auto reduce_by_available_hedge = [&](auto leg_qty) {
+               return leg_qty;
+            };
+            order_qty = reduce_by_available_hedge(order_qty);
+            // price: (nticks into the opposite market)
+            const Book &leg_book =
+                _quoter->_spreader.get_book(leg_data.leg_index());
+            bool leg_book_populated = is_book_populated(
+                leg_book, SpreaderUtils::invert_side(leg_data.trading_side()));
+            if (!leg_book_populated) {
+               LOG(OrderMan, Info,
+                   "QuoterWaitingForSizeStateWorker::_longpath_analyze_and_act "
+                   "(RH), quoting leg book is empty, not firing order");
+               return;
+            }
+            LOG(OrderMan, Info,
+                "QuoterWaitingForSizeStateWorker::_longpath_analyze_and_act "
+                "leg_book: {} k: {}",
+                leg_book, leg_book.security_key());
+            Price order_price = SpreaderUtils::compute_order_price_through_book(
+                leg_book, leg_data.trading_side(), leg_data.tick_size(),
+                n_ticks);
+            LOG(OrderMan, Info,
+                "QuoterWaitingForSizeStateWorker sending order in reverse "
+                "hedging mode key: {} price: {} quantity: {}",
+                _quoter->_key(), order_price, order_qty);
+            _quoter->_place_order(order_price, order_qty);
+            _quoter->_transform_to_WAITING_FOR_IOC();
+         }
+         return;
+      }
+      case bts::spark::ExecutionStrategy::SpreaderType::
+          ExecutionStrategy_SpreaderType_SNIPER: {
+         bts_assert(
+             _quoter->_this_leg_data.leg_params().has_sniper_quote_params());
+         return;
+      }
+      default: {
+         bts_assert(false);
+         return;
+      }
+      }
       Price out_price;
       Quantity out_quantity;
       _quoter->_compute_price_quantity(out_price, out_quantity);
       _quoter->_adjust_for_hidden_order(out_price, out_quantity);
       if (out_quantity.value() > 0) {
+         // only applicable for > 2 legged spreads
          _quoter->_commit_drivers(out_price);
          LOG(OrderMan, Info,
              "QuoterWaitingForSizeStateWorker sending order "
@@ -67,6 +177,7 @@ void QuoterWaitingForSizeStateWorker::_longpath_analyze_and_act() {
          // this is the only place where a quoter order is entered
          _quoter->_place_order(out_price, out_quantity);
          if (_quoter->_hidden_order)
+            // TODO:sniper
             _quoter->_transform_to_WAITING_FOR_IOC();
          else
             _quoter->_transform_to_WAITING_FOR_FIRST_ACK();

src/lib/orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp

@@ -129,13 +129,20 @@ void SpreaderStartStateWorker::on_activate(
              _spreader->_on_child_order_update(os);
           });
       // 5. sub to market data
+      // subsribe to spread book ()
+      auto spr_sk = _spreader->_bookkeeping_data().spread_sec_key();
+      order_services.order_services().subscribe_to_book(
+          spr_sk, [this](const BookUpdateEvent &bk_event) {
+             _spreader->_on_book_update(bk_event);
+          });
       for (size_t l_idx = 0; l_idx < _spreader->_bookkeeping_data().n_legs();
            l_idx++) {
          const LegPropertyData &leg_prop = _spreader->_bookkeeping_data()
                                                .get_leg_data(l_idx)
                                                .leg_property_data();
          LOG(OrderMan, Info,
-             "SpreaderStartStateWorker::on_activate subbing to book sec_key: "
+             "SpreaderStartStateWorker::on_activate subbing to book "
+             "sec_key: "
              "{}",
              leg_prop.security_key());
          order_services.order_services().subscribe_to_book(

src/lib/orderman/orders/spreader_orders/tests/tests_common.h

@@ -9,6 +9,7 @@
 #include "lib/md/PriceSize.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/orderman/securities/ISecurityDB.h"
+#include "lib/sparkproto/SparkProtos.pb.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
 //#include "lib/utils/REQUIRE.h"
@@ -133,6 +134,14 @@ class SpreaderManagerForTest : public ISpreaderManager {
    const bts::orderman::Book &get_book(size_t leg_index) const override {
       return _bookkeeping_data.get_leg_data(leg_index).book();
    }
+   const bts::orderman::Book &get_spread_book() const override {
+      // TODO: since its not used in tests, we just do this ot pass the tests
+      return _bookkeeping_data.spread_book.value();
+      //return get_book(0);
+   }
+   const api::orderman::OrderDefinition &definition() const override {
+      return _bookkeeping_data.order_def();
+   }
    // the caller can keep a reference to the returned type forever. the
    // data in the returned type is all const, so nothing will change
    // beneath it
@@ -158,6 +167,11 @@ class SpreaderManagerForTest : public ISpreaderManager {
       return &rv;
    }
    bool in_cache_warming_cycle() const override { return false; }
+   bts::spark::ExecutionStrategy::SpreaderType
+   get_spreader_quoting_type() const override {
+      return bts::spark::ExecutionStrategy::SpreaderType::
+          ExecutionStrategy_SpreaderType_REQUOTE;
+   }
 
    // SpreaderFractionalQtyHelper &fractional_qty_helper() {
    //    return _fractional_qty_helper;

src/lib/sparkproto/SparkProtos.proto

@@ -1,4 +1,5 @@
-syntax="proto3";
+ 
+syntax="proto3";
 package bts.spark;
 option csharp_namespace="bts.spark";
 
@@ -39,9 +40,11 @@ message LegSpreaderParams
   }
   /// Legs can be quoted actively, or sent in reponse
   /// to fills
-  bool is_active_quoting = 20;
+   bool is_active_quoting = 20;
   bool enable_hedging = 30;
   QuoteParams quote_params = 40;
+  ReverseHedgeQuoteParams reverse_hedge_quote_params = 43;
+  SniperQuoteParams sniper_quote_params = 47;
   PreHedgeParams pre_hedge_params = 50;
   PostHedgeParams post_hedge_params = 60;
   MarketDataType market_data_type = 70;
@@ -52,6 +55,8 @@ message LegSpreaderParams
 //  ******************************************Quote**************************************************
 //  *************************************************************************************************/
 
+
+
 message QuoteParams 
 {
   // Defines how much volume required to price off of a hedge leg level, as a ratio of working leg size.
@@ -77,6 +82,16 @@ message QuoteParams
   NullableQuantity show_quantity = 2010;
 }
 
+message ReverseHedgeQuoteParams{
+  int32 n_ticks = 1;
+}
+
+message SniperQuoteParams{
+  int32 n_ticks = 1;
+}
+
+
+
 /*************************************Quote MuteZone***********************************
 (TT Basic Slop)
 This rule sets a band above and below the quoting order inside of which any price changes will be
@@ -476,6 +491,13 @@ message ExecutionStrategy
     RECALCULATE_CROSSES = 1;
   }
   HedgePricing hedge_pricing = 40;
+  enum SpreaderType
+  {
+    REQUOTE = 0;      // ie send out a quote leaning on the other legs and requotes
+    REVERSE_HEDGE= 1; // sends an sniper IOC on one leg when the spread market is through the desired spread price
+    SNIPER = 2;       // sends simultanious cross orders on all legs when the spread market is through the desired spread price
+  }
+  SpreaderType spreader_type = 50;
 }
 
 // obsolete in api, but still used internally by Spark csproj

src/server/sparkd/CMakeLists.txt

@@ -39,7 +39,7 @@ add_dependencies(${PROJECT_NAME} rapidjson_project)
 FetchContent_Declare(
   databento
   GIT_REPOSITORY https://github.com/databento/databento-cpp
-  GIT_TAG HEAD
+  GIT_TAG v0.31.0
 )
 FetchContent_MakeAvailable(databento)
 

src/server/sparkd/cmds/md/common/VenueOption.cpp

@@ -22,6 +22,7 @@ EnumOptionNames<api::Venue> VenueOption::names() {
        NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES),
        NAME_PAIR(api::VENUE_CBOE),
        NAME_PAIR(api::VENUE_TAIFEX),
+       NAME_PAIR(api::VENUE_TWSE),
        NAME_PAIR(api::VENUE_OKX)};
 }
 
@@ -57,6 +58,8 @@ std::string VenueOption::name(api::Venue venue) {
       return "okx";
    case api::VENUE_TAIFEX:
       return "taifex";
+   case api::VENUE_TWSE:
+      return "twse";
    default:
       return std::to_string(static_cast<int32_t>(venue));
    }

src/server/sparkd/configurations/alphaworx_prod/.env

@@ -1,3 +1,4 @@
 SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_prod
 SPARKD__SHM_SIZE=8gb
 SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/

src/server/sparkd/configurations/alphaworx_zen_hk_live/.env

@@ -0,0 +1,4 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_zen_hk_live
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf/

src/server/sparkd/configurations/alphaworx_zen_hk_live/README.txt

@@ -0,0 +1,6 @@
+* Connects to Binance and Okx PRODUCTION sites for market data and execution
+* Api keys NOT included
+* Configured for Alphaworx
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".

src/server/sparkd/configurations/alphaworx_zen_hk_live/binance_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}

src/server/sparkd/configurations/alphaworx_zen_hk_live/binance_usds_futures_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}

src/server/sparkd/configurations/alphaworx_zen_hk_live/init_env.py

@@ -0,0 +1,88 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx-binance-api-keys"
+    region_name = "ap-northeast-1"
+    profile_name = "AdministratorAccess-849458505639"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_API_KEY"] = secret["api-key"]
+    if "secret-key" in secret:
+        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()

src/server/sparkd/configurations/alphaworx_zen_hk_live/spark.conf

@@ -0,0 +1,124 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.com"
+      url_ws_base: "wss://stream.binance.com:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[OkxSpots].symbol
+        }
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[OkxFutures].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+  }
+}
+
+OkxExeCfg {
+  value {
+    account: "PROD"
+    ops_id: "op_id_bts"
+    secret_key: "240038D7A28D7F9461247CFCB8E43710"
+    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+    ws_url: "wss://wsaws.okx.com:8443"
+    rest_url: "https://www.okx.com"
+    ws_ping_timeout_seconds: 10 
+    ws_ping_interval_seconds: 25
+    ws_reconnect_interval_seconds: 5
+    ws_reconnect_max_count: 5
+	  simulated_trading: false
+    tdmode: "cross"
+  }
+}

src/server/sparkd/configurations/alphaworx_zen_hk_live/symbols.conf

@@ -0,0 +1,124 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "PAXG"
+      symbol: "PEPE"
+      symbol: "S"
+      symbol: "SHIB"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TAO"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "1000PEPE"
+      symbol: "1000SHIB"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "S"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TAO"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "XRP"
+    }
+    list {
+      key: "OkxSpots"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BETH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "PEPE"
+      symbol: "S"
+      symbol: "SHIB"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "USDT"
+      symbol: "XRP"
+    }
+    list {
+      key: "OkxFutures"
+      symbol: "ADA"
+      symbol: "APT"
+      symbol: "AVAX"
+      symbol: "BCH"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "DOT"
+      symbol: "ETH"
+      symbol: "HBAR"
+      symbol: "LINK"
+      symbol: "LTC"
+      symbol: "OM"
+      symbol: "PEPE"
+      symbol: "S"
+      symbol: "SHIB"
+      symbol: "SOL"
+      symbol: "SUI"
+      symbol: "TAO"
+      symbol: "TRUMP"
+      symbol: "USDC"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[OkxSpots].symbol
+      future: @SymbolLists.list[OkxFutures].symbol
+    }
+  }
+}

src/server/sparkd/md/MarketDataGateway.cpp

@@ -38,6 +38,7 @@
 #include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
 #include "server/sparkd/md/recording/MDReplayer.h"
 #include "server/sparkd/md/taifex/TaifexDataGateway.h"
+#include "server/sparkd/md/twse/TwseDataGateway.h"
 
 #include <bits/stdint-uintn.h>
 #include <boost/filesystem.hpp>
@@ -150,8 +151,9 @@ void MarketDataGateway::run() {
       bts_assert(me_->md_config.emumba_deribit().size() == 1);
       _start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
    }
-   start_glbx(me_->md_config);
-   start_taifex(me_->md_config);
+  start_glbx(me_->md_config);
+   start_taifex(me_->md_config); 
+   start_twse(me_->md_config); 
    start_cboe(me_->md_config);
    _start_replays(me_->md_config);
    while (true) {
@@ -399,8 +401,37 @@ void MarketDataGateway::_start_replays(
        os::get_thread_id(), os::get_thread_name());
 }
 
-void MarketDataGateway::start_taifex(
-    const bts::api::md::MarketDataConfig &cfg) {
+void MarketDataGateway::start_twse(const bts::api::md::MarketDataConfig &cfg) {
+   // twse
+//    int counter = 0;
+//         LOG(md, Info, "In start, mdgtway.cpp");
+
+//    for (const auto &twse : cfg.twse()) {  
+//       counter++;
+//       auto p = me_->venue_listeners.emplace_back(
+//           std::make_shared<VenueListenerData>(bts::api::VENUE_TWSE));
+//       std::string tid_name = "twse" + std::to_string(counter);
+//       bts::threading::make_thread(tid_name, [&]() {
+//          bts::md::twse::TwseDataGateway::run(
+//           twse, p->listener, me_->sec_id_store);
+//       }).detach();
+//    }
+    for (const auto &twse : cfg.twse()) {
+      auto p = me_->venue_listeners.emplace_back(
+          std::make_shared<VenueListenerData>(bts::api::VENUE_TWSE));
+          // Capture `p->listener` and `me_->sec_id_store` by value
+      auto listener = p->listener;
+      auto sec_id_store = me_->sec_id_store;
+      std::string tid_name = "twse";
+      bts::threading::make_thread(tid_name, [listener, sec_id_store, twse]() {
+         bts::md::twse::TwseDataGateway::run(
+          twse, listener, sec_id_store);
+      }).detach();
+      
+   }
+}
+
+void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
    // taifex
    for (const auto &taifex : cfg.taifex()) {
       auto p = me_->venue_listeners.emplace_back(

src/server/sparkd/md/MarketDataGateway.h

@@ -39,6 +39,7 @@ class MarketDataGateway {
    void start_glbx(const bts::api::md::MarketDataConfig &cfg);
    void start_cboe(const bts::api::md::MarketDataConfig &cfg);
    void start_taifex(const bts::api::md::MarketDataConfig &cfg);
+   void start_twse(const bts::api::md::MarketDataConfig &cfg);
 };
 
 } // namespace bts::md

src/server/sparkd/md/okx/OkxInstrumentStore.cpp

@@ -56,20 +56,25 @@ static void instrument_to_sec(const OkxInstrument &oi, Security &sec) {
    case bts::md::okx::OkxInstrumentType::SWAP:{
       auto &fut = sec.security.emplace<FutureSecurity>();
       core::TimeStamp::from_ymd(3000, 1, 1,&fut.expiration);
+      sec.security_common.symbol = oi.settleCcy;
+      scomm.cash_per_point = oi.ctVal;
       if (oi.instId.find("USDT") != std::string::npos ||
           oi.instId.find("USDC") != std::string::npos) {
          sec.security_common.symbol =
              oi.instId.substr(0, oi.instId.find_first_of("-"));
-         scomm.cash_per_point = oi.ctVal;
-         break;
       }
-      sec.security_common.symbol = oi.settleCcy;
       break;
    }
    case OkxInstrumentType::FUTURES: {
-      sec.security_common.symbol = oi.settleCcy;
       auto &fut = sec.security.emplace<FutureSecurity>();
       fut.expiration = *oi.expTime;
+      sec.security_common.symbol = oi.settleCcy;
+      scomm.cash_per_point = oi.ctVal;
+      if (oi.instId.find("USDT") != std::string::npos ||
+            oi.instId.find("USDC") != std::string::npos) {
+            sec.security_common.symbol =
+               oi.instId.substr(0, oi.instId.find_first_of("-"));
+      }
       break;
    }
    case OkxInstrumentType::OPTION: {

src/server/sparkd/md/recording/DbWriters.h

@@ -26,6 +26,12 @@ class DbWriters {
          LOG(md, Info, "{}", bts::utils::pb::to_string(rec_msg));
       }
    }
+   // to check subset values for each record
+   static void dumpSubsets(const std::vector<DataToRecord>& data) {
+      for (const auto &r : data) {
+          LOG(md, Info, "Record subset: '{}'", r.subset());
+      }
+  }
 
    // Writes data to an SQL database
    static std::unique_ptr<SQLite::Database> open_db(
@@ -69,6 +75,12 @@ class DbWriters {
       db->exec("CREATE TABLE IF NOT EXISTS trades ("
                "ts TEXT NOT NULL, key INTEGER NOT NULL, price float NOT NULL, "
                "quantity float NOT NULL, total_volume );");
+      db->exec("CREATE TABLE IF NOT EXISTS tbbo ("
+               "ts TEXT, key INTEGER, price float, "
+               "quantity float, total_volume float, book_ts TEXT, "
+               "book_key INTEGER, "
+               "bid_price_1 float, bid_qty_1 float, ask_price_1 float, "
+               "ask_qty_1 float);");
 
       return db;
    }
@@ -84,47 +96,242 @@ class DbWriters {
       }
       return oss.str();
    }
+
+   static void writeToSQL(const std::vector<DataToRecord> &data) {
+      LOG(md, Info, "Writting {} entries to sql", data.size());
+
+      dumpSubsets(data);
+      
+      // Open the database.
+      std::unique_ptr<SQLite::Database> db = open_db();
+      
+      try {
+         // Group records by the subset field.
+         std::map<std::string, std::vector<DataToRecord>> subsets;
+         for (const auto &r : data) {
+            std::string subset = r.subset();  // subset field from the proto
+            if (!subsets.count(subset)) {
+               subsets[subset] = std::vector<DataToRecord>();
+            }
+            subsets[subset].push_back(r);
+         }
+   
+         // Begin a transaction for bulk insert.
+         SQLite::Transaction transaction(*db);
+   
+         // Process each group.
+         for (const auto &pair : subsets) {
+            const std::string &tableName = pair.first;  // dynamic table name from subset
+            const auto &records = pair.second;
+            
+            // First, determine the table schema based on record type and create the table if needed.
+            std::string createTableQuery;
+            if (!records.empty() && records.front().has_total_book()) {
+               createTableQuery = "CREATE TABLE IF NOT EXISTS " + tableName + " ("
+                                  "ts TEXT NOT NULL, key INTEGER NOT NULL, "
+                                  "bid_price_1 float, bid_qty_1 float, ask_price_1 float, ask_qty_1 float, "
+                                  "bid_price_2 float, bid_qty_2 float, ask_price_2 float, ask_qty_2 float, "
+                                  "bid_price_3 float, bid_qty_3 float, ask_price_3 float, ask_qty_3 float, "
+                                  "bid_price_4 float, bid_qty_4 float, ask_price_4 float, ask_qty_4 float, "
+                                  "bid_price_5 float, bid_qty_5 float, ask_price_5 float, ask_qty_5 float, "
+                                  "bid_price_6 float, bid_qty_6 float, ask_price_6 float, ask_qty_6 float, "
+                                  "bid_price_7 float, bid_qty_7 float, ask_price_7 float, ask_qty_7 float, "
+                                  "bid_price_8 float, bid_qty_8 float, ask_price_8 float, ask_qty_8 float, "
+                                  "bid_price_9 float, bid_qty_9 float, ask_price_9 float, ask_qty_9 float, "
+                                  "bid_price_10 float, bid_qty_10 float, ask_price_10 float, ask_qty_10 float)";
+            } else if (!records.empty() && records.front().has_trade()) {
+               createTableQuery = "CREATE TABLE IF NOT EXISTS " + tableName + " ("
+                                  "ts TEXT , key INTEGER , price float , "
+                                  "quantity float , total_volume float)";
+            } else if (!records.empty() && records.front().has_bbo_trade()) {
+               createTableQuery = "CREATE TABLE IF NOT EXISTS " + tableName + " ("
+                                  "ts TEXT, key INTEGER, price float , "
+                                  "quantity float, total_volume float, book_ts TEXT, "
+                                  "book_key INTEGER, "
+                                  "bid_price_1 float, bid_qty_1 float, ask_price_1 float, ask_qty_1 float)";
+            } else {
+               continue;
+            }
+            
+            // Execute the CREATE TABLE query.
+            {
+               SQLite::Statement createStmt(*db, createTableQuery);
+               createStmt.exec();
+            }
+            
+            // Now build the INSERT query.
+            std::string insertQuery;
+            if (!records.empty() && records.front().has_total_book()) {
+               insertQuery = "INSERT OR REPLACE INTO " + tableName +
+                             " (ts, key, bid_price_1, bid_qty_1, ask_price_1, ask_qty_1, "
+                             "bid_price_2, bid_qty_2, ask_price_2, ask_qty_2, "
+                             "bid_price_3, bid_qty_3, ask_price_3, ask_qty_3, "
+                             "bid_price_4, bid_qty_4, ask_price_4, ask_qty_4, "
+                             "bid_price_5, bid_qty_5, ask_price_5, ask_qty_5, "
+                             "bid_price_6, bid_qty_6, ask_price_6, ask_qty_6, "
+                             "bid_price_7, bid_qty_7, ask_price_7, ask_qty_7, "
+                             "bid_price_8, bid_qty_8, ask_price_8, ask_qty_8, "
+                             "bid_price_9, bid_qty_9, ask_price_9, ask_qty_9, "
+                             "bid_price_10, bid_qty_10, ask_price_10, ask_qty_10)";
+            } else if (!records.empty() && records.front().has_trade()) {
+               insertQuery = "INSERT OR REPLACE INTO " + tableName +
+                             " (ts, key, price, quantity, total_volume)";
+            } else if (!records.empty() && records.front().has_bbo_trade()) {
+               insertQuery = "INSERT OR REPLACE INTO " + tableName +
+                             " (ts, key, price, quantity, total_volume, book_ts, book_key, "
+                             "bid_price_1, bid_qty_1, ask_price_1, ask_qty_1)";
+            }
+            
+            // Build value tuples.
+            std::vector<std::string> valueStrings;
+            for (const auto &entry : records) {
+               std::string tuple;
+               if (entry.has_total_book()) {
+                  auto book = entry.total_book();
+                  tuple = "(" + std::to_string(entry.timestamp()) + ", " +
+                          std::to_string(entry.sec_key());
+                  const auto &asks = book.direct_book().asks();
+                  const auto &bids = book.direct_book().bids();
+                  for (int i = 0; i < 10; ++i) {
+                     // Process asks.
+                     if (i < asks.size()) {
+                        IntPrice p(asks.at(i).mantissa(), asks.at(i).exponent());
+                        tuple += ", " + std::to_string(p.price()) + ", " +
+                                 std::to_string(asks.at(i).size());
+                     } else {
+                        tuple += ", NULL, NULL";
+                     }
+                     // Process bids.
+                     if (i < bids.size()) {
+                        IntPrice p(bids.at(i).mantissa(), bids.at(i).exponent());
+                        tuple += ", " + std::to_string(p.price()) + ", " +
+                                 std::to_string(bids.at(i).size());
+                     } else {
+                        tuple += ", NULL, NULL";
+                     }
+                  }
+                  tuple += ")";
+               } else if (entry.has_trade()) {
+                  const auto &trade = entry.trade();
+                  IntPrice p(trade.mantissa(), trade.exponent());
+                  tuple = "(" + std::to_string(entry.timestamp()) + ", " +
+                          std::to_string(entry.sec_key()) + ", " +
+                          std::to_string(p.price()) + ", " +
+                          std::to_string(trade.size()) + ", " +
+                          std::to_string(trade.total_volume()) + ")";
+               } else if (entry.has_bbo_trade()) {
+                  const auto &bbo_trade = entry.bbo_trade();
+                  const auto &trade = bbo_trade.trade();
+                  IntPrice p(trade.mantissa(), trade.exponent());
+                  
+                  tuple = "(" 
+                      + std::to_string(trade.time()) + ", " 
+                      + std::to_string(trade.key()) + ", " 
+                      + std::to_string(p.price()) + ", " 
+                      + std::to_string(trade.size()) + ", " 
+                      + std::to_string(trade.total_volume()) + ", "
+                      + std::to_string(entry.timestamp()) + ", " 
+                      + std::to_string(entry.sec_key());
+              
+                  const auto &asks = bbo_trade.book().direct_book().asks(); 
+                  const auto &bids = bbo_trade.book().direct_book().bids();
+              
+                  // Process top ask (level 1)
+                  if (!asks.empty()) {
+                      IntPrice askPrice(asks[0].mantissa(), asks[0].exponent());
+                      tuple += ", " + std::to_string(askPrice.price()) + ", " 
+                              + std::to_string(asks[0].size());  
+                  } else {
+                      tuple += ", NULL, NULL";
+                  }
+              
+                  // Process top bid (level 1)
+                  if (!bids.empty()) {
+                      IntPrice bidPrice(bids[0].mantissa(), bids[0].exponent());
+                      tuple += ", " + std::to_string(bidPrice.price()) + ", " 
+                              + std::to_string(bids[0].size()); 
+                  } else {
+                      tuple += ", NULL, NULL";
+                  }
+              
+                  tuple += ")";
+              }
+              
+               valueStrings.push_back(tuple);
+            }
+            std::string fullQuery = insertQuery + " VALUES " + join(valueStrings, ", ") + ";";
+            
+            // Replace any occurrence of "nan" with "NULL".
+            auto replaceNanWithNull = [](std::string &input) {
+               const std::string target = "nan";
+               const std::string replacement = "NULL";
+               size_t pos = 0;
+               while ((pos = input.find(target, pos)) != std::string::npos) {
+                  input.replace(pos, target.length(), replacement);
+                  pos += replacement.length();
+               }
+            };
+            replaceNanWithNull(fullQuery);
+            LOG(md, Diag, "EXECUTING QUERY '{}'", fullQuery);
+            SQLite::Statement q(*db, fullQuery);
+            q.exec();
+         }
+         transaction.commit();
+      } catch (const SQLite::Exception &e) {
+         LOG(md, Warn, "Error writing to books db (retry pending): {}",
+             std::string(e.what()));
+      }
+   }
+   
+
+   /*old writeToSQL
    static void writeToSQL(const std::vector<DataToRecord> &data) {
       LOG(md, Info, "Writting {} entries to sql", data.size());
       std::unique_ptr<SQLite::Database> db = open_db();
 
       try {
          auto db = open_db();
-         std::string insert_book = "INSERT OR REPLACE INTO books (ts, key, "
-                  "bid_price_1, bid_qty_1, ask_price_1, ask_qty_1, "
-                  "bid_price_2, bid_qty_2, ask_price_2, ask_qty_2, "
-                  "bid_price_3, bid_qty_3, ask_price_3, ask_qty_3, "
-                  "bid_price_4, bid_qty_4, ask_price_4, ask_qty_4, "
-                  "bid_price_5, bid_qty_5, ask_price_5, ask_qty_5, "
-                  "bid_price_6, bid_qty_6, ask_price_6, ask_qty_6, "
-                  "bid_price_7, bid_qty_7, ask_price_7, ask_qty_7, "
-                  "bid_price_8, bid_qty_8, ask_price_8, ask_qty_8, "
-                  "bid_price_9, bid_qty_9, ask_price_9, ask_qty_9, "
-                  "bid_price_10, bid_qty_10, ask_price_10, ask_qty_10)";
-         std::string insert_trade= "INSERT OR REPLACE INTO trades (ts, key, price, quantity, total_volume)";
+         std::string insert_book =
+             "INSERT OR REPLACE INTO books (ts, key, "
+             "bid_price_1, bid_qty_1, ask_price_1, ask_qty_1, "
+             "bid_price_2, bid_qty_2, ask_price_2, ask_qty_2, "
+             "bid_price_3, bid_qty_3, ask_price_3, ask_qty_3, "
+             "bid_price_4, bid_qty_4, ask_price_4, ask_qty_4, "
+             "bid_price_5, bid_qty_5, ask_price_5, ask_qty_5, "
+             "bid_price_6, bid_qty_6, ask_price_6, ask_qty_6, "
+             "bid_price_7, bid_qty_7, ask_price_7, ask_qty_7, "
+             "bid_price_8, bid_qty_8, ask_price_8, ask_qty_8, "
+             "bid_price_9, bid_qty_9, ask_price_9, ask_qty_9, "
+             "bid_price_10, bid_qty_10, ask_price_10, ask_qty_10)";
+         std::string insert_trade = "INSERT OR REPLACE INTO trades (ts, key, "
+                                    "price, quantity, total_volume)";
+         std::string insert_tbbo = "INSERT OR REPLACE INTO tbbo (ts, key, "
+                                   "price, quantity, total_volume, "
+                                   "book_ts, book_key, bid_price_1, bid_qty_1, "
+                                   "ask_price_1, ask_qty_1)";
          SQLite::Transaction transaction(*db);
 
-         auto replaceNanWithNull = [](std::string& input) {
-        const std::string target = "nan";
-        const std::string replacement = "NULL";
-        size_t pos = 0;
+         auto replaceNanWithNull = [](std::string &input) {
+            const std::string target = "nan";
+            const std::string replacement = "NULL";
+            size_t pos = 0;
 
-        while ((pos = input.find(target, pos)) != std::string::npos) {
-            input.replace(pos, target.length(), replacement);
-            pos += replacement.length(); // Move past the replacement
-        }
-    };
+            while ((pos = input.find(target, pos)) != std::string::npos) {
+               input.replace(pos, target.length(), replacement);
+               pos += replacement.length(); // Move past the replacement
+            }
+         };
          // Collect data for bulk insert
          std::vector<std::string> book_values;
          std::vector<std::string> trade_values;
+         std::vector<std::string> tbbo_values;
 
          for (const auto &entry : data) {
             if (entry.has_total_book()) {
                auto book = entry.total_book();
                std::string &book_value = book_values.emplace_back();
-               book_value = "(" +
-                                        std::to_string(entry.timestamp()) +
-                                        ", " + std::to_string(entry.sec_key());
+               book_value = "(" + std::to_string(entry.timestamp()) + ", " +
+                            std::to_string(entry.sec_key());
 
                const auto &asks = book.direct_book().asks();
                const auto &bids = book.direct_book().bids();
@@ -146,7 +353,7 @@ class DbWriters {
                      book_value += ", NULL, NULL";
                   }
                }
-               book_value +=")";
+               book_value += ")";
             }
             if (entry.has_trade()) {
                const auto &trade = entry.trade();
@@ -158,11 +365,44 @@ class DbWriters {
                    std::to_string(trade.size()) + ", " +
                    std::to_string(trade.total_volume()) + ")");
             }
+            if (entry.has_bbo_trade()) {
+               const auto &bbo_trade = entry.bbo_trade();
+               IntPrice p(bbo_trade.trade().mantissa(), bbo_trade.trade().exponent());
+               std::string tbbo_value =
+                   "(" + std::to_string(entry.timestamp()) + ", " +
+                   std::to_string(entry.sec_key()) + ", " +
+                   std::to_string(p.price()) +", " + 
+                   std::to_string(bbo_trade.trade().size()) + ", " +
+                   std::to_string(bbo_trade.trade().total_volume());
+
+               const auto &asks = bbo_trade.book().direct_book().asks(); 
+               const auto &bids = bbo_trade.book().direct_book().bids();
+
+               if (!asks.empty()) {
+                  IntPrice askPrice(asks.begin()->mantissa(), asks.begin()->exponent());
+                  tbbo_value += ", " + std::to_string(askPrice.price()) + ", " +
+                                std::to_string(asks.size());
+               } else {
+                  tbbo_value += ", NULL, NULL";
+               }
+
+               if (!bids.empty()) {
+                  IntPrice bidPrice(
+                      bids.begin()->mantissa(), bids.begin()->exponent());
+                  tbbo_value += ", " + std::to_string(bidPrice.price()) + ", " +
+                                std::to_string(bids.size());
+               } else {
+                  tbbo_value += ", NULL, NULL";
+               }
+
+               tbbo_value += ")";
+               tbbo_values.push_back(tbbo_value);
+            }
          }
          // Construct the full bulk insert query for books
          if (!book_values.empty()) {
             std::string book_query =
-                insert_book + " VALUES "+ join(book_values, ", ")+";";
+                insert_book + " VALUES " + join(book_values, ", ") + ";";
             replaceNanWithNull(book_query);
             LOG(md, Diag, "EXECUTING QUERY '{}'", book_query);
             SQLite::Statement q(*db, book_query);
@@ -172,7 +412,7 @@ class DbWriters {
          // Construct the full bulk insert query for trades
          if (!trade_values.empty()) {
             std::string trade_query =
-                insert_trade + " VALUES "+ join(trade_values, ", ")+";";
+                insert_trade + " VALUES " + join(trade_values, ", ") + ";";
             replaceNanWithNull(trade_query);
             LOG(md, Diag, "EXECUTING QUERY '{}'", trade_query);
             SQLite::Statement q(*db, trade_query);
@@ -180,13 +420,23 @@ class DbWriters {
             ;
          }
 
+         if (!tbbo_values.empty()) {
+            std::string tbbo_query =
+                insert_tbbo + " VALUES " + join(tbbo_values, ", ") + ";";
+            replaceNanWithNull(tbbo_query);
+            LOG(md, Diag, "EXECUTING QUERY '{}'", tbbo_query);
+            SQLite::Statement q(*db, tbbo_query);
+            q.exec();
+            ;
+         }
+
          transaction.commit();
       } catch (const SQLite::Exception &e) {
          LOG(md, Warn, "Error writing to books db (retry pending): {}",
              std::string(e.what()));
       }
    }
-
+*/
    // Writes data to a file
    static void writeProtoToFile(
        const std::vector<DataToRecord> &data,

src/server/sparkd/md/recording/MDRecorder.cpp

@@ -30,6 +30,7 @@
 #include <string>
 #include <thread>
 #include <vector>
+#include <cmath>
 
 namespace bts::md::recording {
 
@@ -98,11 +99,13 @@ void copy_misc(IntPrice settlement, MiscData &misc, co2::MiscMessage *pmisc) {
 // each security is represented here
 struct MDRecordingSecEntry {
    SecurityKey key;
+   std::string subset_string;  // New: computed base subset (e.g. "taifex_future_tx")
    MDConsumerHandle handle;
    TimeStamp direct_book_ts = {};
    TimeStamp implied_book_ts = {};
    TimeStamp trade_ts = {};
    TimeStamp misc_ts = {};
+   co2::TotalBook last_total_book;
    MDRecordingSecEntry(SecurityKey key, MDConsumerHandle handle)
        : key(key), handle(handle) {}
 };
@@ -208,6 +211,7 @@ void MDRecorder::_poll_md() {
          MDRecordingSecEntry &md_entry = kv.second;
          if (me->settings.n_book_levels > 0)
             _poll_books(md_entry);
+         _poll_bbotrade(md_entry);  // Added this to poll tbbo
          _poll_trades(md_entry);
          _poll_misc(md_entry);
       }
@@ -231,6 +235,37 @@ void MDRecorder::_poll_md() {
        bts::api::Venue_Name(me->venue));
 }
 
+std::string computeBaseSubsetString(const bts::api::Security &sec) {
+   std::string exchange_str = bts::api::Venue_Name(sec.exchange());
+   
+   std::string instrument_type;
+   if (sec.has_future()) {
+       instrument_type = "future";
+   } else if (sec.has_option()) {
+       instrument_type = "option";
+   } else if (sec.has_spot()) {
+       instrument_type = "spot";
+   } else if (sec.has_spread()) {
+       instrument_type = "spread";
+   } else if (sec.has_synthetic()) {
+       instrument_type = "synthetic";
+   } else {
+       instrument_type = "unset";
+   }
+   //might need to add toLowerCase for exact match to sheet requirement 
+   std::string symbol = sec.symbol();
+
+
+   // Convert instrument_type and symbol to lowercase
+   std::transform(exchange_str.begin(), exchange_str.end(), exchange_str.begin(), ::tolower);
+   std::transform(instrument_type.begin(), instrument_type.end(), instrument_type.begin(), ::tolower);
+   std::transform(symbol.begin(), symbol.end(), symbol.begin(), ::tolower);
+
+
+   // This base string does not include the event prefix (like "book", "tradebbo", etc.)
+   return exchange_str + "_" + instrument_type + "_" + symbol;
+}
+
 void MDRecorder::_scan_securities() {
    auto secs = me->md_client.current_consumer().get_securities();
    uint32_t n_secs_added = 0;
@@ -243,14 +278,26 @@ void MDRecorder::_scan_securities() {
          LOG(md, Info,
              "MDRecorder::_scan_securities adding sec: {}, n_secs: {}", key,
              me->handles.size());
-         me->handles.emplace(
+          
+          // Compute the base subset string from the security details.
+          std::string baseSubset = computeBaseSubsetString(*s);
+          // Create a new MDRecordingSecEntry and assign the computed subset_string.
+          MDRecordingSecEntry entry(key, me->md_client.current_consumer().get_handle(key.value()));
+          entry.subset_string = baseSubset;
+          // Insert the entry with the computed subset_string.
+          me->handles.emplace(key, std::move(entry));
+
+         /*me->handles.emplace(
              key, MDRecordingSecEntry(
                       key, me->md_client.current_consumer().get_handle(
-                               key.value())));
+                               key.value())));*/
          bts::co2::RecordedMDMessage rec_msg;
          rec_msg.set_sec_key(key.value());
          rec_msg.set_timestamp(bts::core::TimeStamp::now_ns());
          rec_msg.mutable_security()->CopyFrom(*s);
+
+         rec_msg.set_subset(baseSubset);
+
          _write_pb_to_file(rec_msg);
          n_secs_added++;
       }
@@ -298,6 +345,10 @@ void MDRecorder::_poll_books(MDRecordingSecEntry &md_entry) {
    if (md_change) {
       co2::RecordedMDMessage rec_msg;
       rec_msg.set_sec_key(md_entry.key.value());
+      //for tbbo consistency
+      std::string eventSubset = "book_" + md_entry.subset_string;
+       rec_msg.set_subset(eventSubset);
+       //--------
       rec_msg.set_timestamp(bts::core::TimeStamp::now_ns());
       rec_msg.mutable_total_book()->CopyFrom(book);
       _write_pb_to_file(rec_msg);
@@ -325,13 +376,65 @@ void MDRecorder::_poll_trades(MDRecordingSecEntry &md_entry) {
       // aggressor?
       co2::RecordedMDMessage rec_msg;
       rec_msg.set_sec_key(md_entry.key.value());
+      //for tbbo consistency 
+      std::string eventSubset = "trade_" + md_entry.subset_string;
+       rec_msg.set_subset(eventSubset);
+       //-----
       rec_msg.set_timestamp(bts::core::TimeStamp::now_ns());
       rec_msg.mutable_trade()->CopyFrom(tm);
       _write_pb_to_file(rec_msg);
       _write_pb_to_db(rec_msg);
    }
 }
+void MDRecorder::_poll_bbotrade(MDRecordingSecEntry &md_entry) {
+   // === Always Update Book Snapshot ===
+   TimeStamp book_ts;
+   co2::TotalBook book;
+   me->reusables.clean_books();
+   md_entry.handle.get_direct_book(book_ts, me->settings.n_book_levels,
+                                   me->reusables.bids, me->reusables.asks);
+   
+   // Update book snapshot even if timestamp hasn't changed (for TBBO accuracy)
+   co2::BookMessage *direct_book = book.mutable_direct_book();
+   copy_book(me->reusables.bids, true, me->settings.n_book_levels, *direct_book);
+   copy_book(me->reusables.asks, false, me->settings.n_book_levels, *direct_book);
+   md_entry.last_total_book.CopyFrom(book);  // Always update
+   md_entry.direct_book_ts = book_ts;        // Track timestamp
 
+   // === Trade Handling ===
+   TimeStamp trade_ts;
+   PriceSize t_ps;
+   datetime::LLDateTime trade_time;
+   size_t total_volume;
+   int from_spread;
+   md_entry.handle.get_last_trade(trade_ts, t_ps, trade_time, &total_volume, &from_spread);
+
+   if (trade_ts > md_entry.trade_ts) {
+       md_entry.trade_ts = trade_ts;
+
+       co2::RecordedMDMessage rec_msg;
+       rec_msg.set_sec_key(md_entry.key.value());
+       rec_msg.set_timestamp(trade_time);
+       rec_msg.set_subset("tradebbo_" + md_entry.subset_string);
+       auto* bboTrade = rec_msg.mutable_bbo_trade();
+       bboTrade->set_event_type(co2::TRADE);
+       
+       // Directly populate the trade (no need for TradeMessage intermediary)
+       auto* trade = bboTrade->mutable_trade();
+       trade->set_key(md_entry.key.value());
+       trade->set_time(trade_time);
+       trade->set_mantissa(t_ps.mantissa());
+       trade->set_exponent(t_ps.exponent());
+       trade->set_size(t_ps.direct_size());
+       trade->set_total_volume(total_volume);
+
+       // Attach the freshly updated book snapshot
+       bboTrade->mutable_book()->CopyFrom(md_entry.last_total_book);
+
+       _write_pb_to_file(rec_msg);
+       _write_pb_to_db(rec_msg);
+   }
+}
 void MDRecorder::_poll_misc(MDRecordingSecEntry &md_entry) {
    auto misc_data = md_entry.handle.misc();
    if (misc_data.misc_timestamp > md_entry.misc_ts) {
@@ -398,4 +501,4 @@ bool MDRecorder::is_md_recording_file(
           file_name.find(search_txt) != std::string::npos;
 }
 
-} // namespace bts::md::recording
+} // namespace bts::md::recording

src/server/sparkd/md/recording/MDRecorder.h

@@ -30,6 +30,7 @@ class MDRecorder {
    void _scan_securities();
    void _poll_books(MDRecordingSecEntry &md_entry);
    void _poll_trades(MDRecordingSecEntry &md_entry);
+   void _poll_bbotrade(MDRecordingSecEntry &md_entry);
    void _poll_misc(MDRecordingSecEntry &md_entry);
    void _write_pb_to_file(const bts::co2::RecordedMDMessage &rec_msg);
    void _write_pb_to_db(const bts::co2::RecordedMDMessage &rec_msg);

src/server/sparkd/md/taifex/TaifexDataGateway.cpp

@@ -10,6 +10,10 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/overloaded.h"
 #include "lib/utils/thread_manager.h"
+#include <chrono>
+#include <dirent.h>
+#include <dirent.h> // For opendir, readdir, closedir
+#include <iostream> // For logging
 #include <memory>
 #include <optional>
 #include <ostream>
@@ -98,6 +102,31 @@ void handleReferenceDataUnparsedMessage(
    auto parsed = getParsedReferenceMessage(buffer, position, Header);
    if (!parsed.has_value())
       return;
+   ///adding temp code to write to preload reference file
+   // std::set<std::string> I010_seen; // Temporary sets for this run
+   //  std::set<std::string> I011_seen;
+   //  bool seen = true;
+   //  std::visit(
+   //      overloaded{
+   //          [&](const TaifexMessages::I010Data &i010) {
+   //              if (!I010_seen.contains(i010.prod_id) &&
+   //                  TaifexUtils::sec_filter(i010.prod_id)) {
+   //                  seen = false;
+   //                  I010_seen.insert(i010.prod_id);
+   //              }
+   //          },
+   //          [&](const TaifexMessages::I011Data &i011) {
+   //              if (!I011_seen.contains(i011.kind_id) &&
+   //                  TaifexUtils::sec_filter(i011.kind_id)) {
+   //                  seen = false;
+   //                  I011_seen.insert(i011.kind_id);
+   //              }
+   //          }},
+   //      parsed.value());
+   //  if (!seen) {
+   //      writeBufferWithSeparator(buffer, Header.Body_Length + MSG_HEADER_SIZE + MSG_FOOTER_SIZE, position);
+   //  }
+   /////temp code end
    handleReferenceDataParsedMessage(
        parsed.value(), listener, sec_id_store, security_details);
 }
@@ -329,6 +358,7 @@ void writeBufferWithSeparator(
     unsigned long position = 0) {
    auto filename = config::get<bts::config::SystemConfig>().data_path() +
                    "taifex_default_reference";
+   // "taifex_default_reference_new_secs";
    std::ofstream outputFile(
        filename, std::ios::binary | std::ios::app); // Open file in append mode
    if (!outputFile) {
@@ -359,6 +389,7 @@ void HandleReferenceFeed(
    LoadReferenceFromFile(
        config::get<bts::config::SystemConfig>().data_path() +
            "taifex_default_reference",
+       // "taifex_default_reference_new_secs",
        snapshot_buffered_messages, I010_seen, I011_seen);
    // bool file_done =
    UNPROCESSED_MSG buffer(FEED_BUFFER_SIZE);
@@ -445,6 +476,116 @@ void HandleLiveBufferFeed(
    }
    stop_live_buffer_thread = true;
 }
+//runOnBinary with debuggin code to find md gaps 
+// bool runOnBinaryFile(
+//    const std::string &file_name,
+//    std::shared_ptr<bts::md::IMarketDataListener> listener,
+//    std::shared_ptr<persistence::IDGenerator> sec_id_store,
+//    std::shared_ptr<TaifexSecurityDetailsMapper> security_details,
+//    TaifexDataGateway::DATA_TYPE data_type,
+//    unsigned long& out_total_processed,  // Output: messages processed this run
+//    int& out_gap_count) {                // Output: gaps found this run
+//    const size_t BUFFER_SIZE = 1024 * 1024;
+//    std::ifstream file(file_name, std::ios::binary);
+//    if (!file) {
+//        LOG(md, Error, "File not found: {}", file_name);
+//        return false;
+//    }
+
+//    // MODIFICATION 1: Open a file to log gaps, in append mode to collect across all files
+//    std::ofstream gap_file("/home/bts/sparkdata/data/gaps_report.txt", std::ios::app);
+//    if (!gap_file) {
+//        LOG(md, Error, "Failed to open gaps_report.txt for writing");
+//    }
+
+//    UNPROCESSED_MSG buffer(BUFFER_SIZE + 1024);
+//    UNPROCESSED_MSG prev_data(1024);
+//    bool start = true;
+//    size_t prev_data_len = 0;
+
+//    unsigned long last_seq_num = 0;
+//    int last_channel_id = -1;
+//    unsigned long total_processed = 0;
+//    static int gap_count = 0;
+//    int initial_gap_count = gap_count;  // Store starting point for this run
+
+//    while (file.read(reinterpret_cast<char *>(buffer.data() + prev_data_len), BUFFER_SIZE)) {
+//        size_t bytes_read = file.gcount();
+//        if (bytes_read < BUFFER_SIZE) {
+//            break;
+//        }
+//        if (!start) {
+//            std::memcpy(buffer.data(), prev_data.data(), prev_data_len);
+//        }
+//        size_t position = 0;
+//        while (position < (BUFFER_SIZE + prev_data_len - MSG_HEADER_SIZE)) {
+//            auto Header = TaifexMessages::Header_Fields(buffer, position);
+//            size_t message_length = Header.Body_Length + MSG_HEADER_SIZE + MSG_FOOTER_SIZE;
+//            if ((BUFFER_SIZE + prev_data_len - position) >= message_length) {
+//                if (!Header.valid) {
+//                    LOG(md, Error, "Invalid header at position {}", position);
+//                    position += message_length;
+//                    continue;
+//                }
+//                switch (data_type) {
+//                case TaifexDataGateway::DATA_TYPE::LIVE:
+//                    LOG(md, Info, "Processed LIVE sequence: {}", Header.Channel_Seq);
+//                    total_processed++;
+//                    if (Header.Channel_Seq != last_seq_num + 1 && last_seq_num != 0) {
+//                        gap_count++;
+//                        LOG(md, Warn, "Gap #{} in recorded LIVE data: {} to {}", 
+//                            gap_count, last_seq_num, Header.Channel_Seq);
+//                        LOG(md, Warn, "header.channel_seq {}", Header.Channel_Seq);
+//                        LOG(md, Warn, "header.channel_id {}", Header.Channel_Id);
+//                        LOG(md, Warn, "last_seq_number {}", last_seq_num);
+//                        LOG(md, Warn, "last channel id  {}", last_channel_id);
+
+//                        if (gap_file) {
+//                         gap_file << "File: " << file_name << "\n";
+//                         gap_file << "Gap #" << gap_count << ": " << last_seq_num 
+//                                  << " to " << Header.Channel_Seq << "\n";
+//                         gap_file << "  header.channel_seq: " << Header.Channel_Seq << "\n";
+//                         gap_file << "  header.channel_id: " << Header.Channel_Id << "\n";
+//                         gap_file << "  last_seq_number: " << last_seq_num << "\n";
+//                         gap_file << "  last channel id: " << last_channel_id << "\n\n";
+//                     }
+//                    }
+//                    last_seq_num = Header.Channel_Seq;
+//                    last_channel_id = Header.Channel_Id;
+//                    handleLiveDataUnparsedMessage(
+//                        buffer, position, Header, listener, security_details);
+//                    break;
+//                case TaifexDataGateway::DATA_TYPE::REFERENCE:
+//                    handleReferenceDataUnparsedMessage(
+//                        buffer, position, Header, listener, sec_id_store, security_details);
+//                    break;
+//                case TaifexDataGateway::DATA_TYPE::REFRESH:
+//                    handleRefreshDataUnparsedMessage(buffer, position, Header);
+//                    break;
+//                default:
+//                    bts_assert(false);
+//                }
+//                position += message_length;
+//            } else {
+//                break;
+//            }
+//        }
+//        prev_data_len = BUFFER_SIZE + prev_data_len - position;
+//        if (prev_data_len > 0) {
+//            std::memcpy(prev_data.data(), buffer.data() + position, prev_data_len);
+//        }
+//        start = false;
+//    }
+
+//    // Set output parameters
+//    out_total_processed = total_processed;
+//    out_gap_count = gap_count - initial_gap_count;  // Gaps for this run only
+
+//    LOG(md, Info, "Finished processing {}: last_seq_num {}, total_processed {}, total_gaps {}", 
+//        file_name, last_seq_num, total_processed, out_gap_count);
+//    return true;
+// }
+
 bool runOnBinaryFile(
     const std::string &file_name,
     std::shared_ptr<bts::md::IMarketDataListener> listener,
@@ -463,6 +604,13 @@ bool runOnBinaryFile(
    UNPROCESSED_MSG prev_data(1024); // Stores leftover data
    bool start = true;
    size_t prev_data_len = 0; // Length of leftover data
+
+   //changes to check for gaps in live data
+   unsigned long last_seq_num = 0; // Track sequence for LIVE data
+   int last_channel_id = -1;
+   static int gap_count = 0;
+   unsigned long total_processed = 0;
+
    while (file.read(
        reinterpret_cast<char *>(buffer.data() + prev_data_len), BUFFER_SIZE)) {
       size_t bytes_read = file.gcount();
@@ -485,6 +633,25 @@ bool runOnBinaryFile(
             // Handle the parsed message
             switch (data_type) {
             case TaifexDataGateway::DATA_TYPE::LIVE:
+
+            LOG(md, Info, "Processed LIVE sequence: {}", Header.Channel_Seq);
+                    total_processed++;
+
+               //adding sequence number check
+               if (Header.Channel_Seq != last_seq_num + 1 &&
+                   last_seq_num != 0) {
+                  gap_count++;
+                  LOG(md, Warn, "Gap #{} in recorded LIVE data: {} to {}",
+                      gap_count, last_seq_num, Header.Channel_Seq);
+                  LOG(md, Warn, "Gap in recorded LIVE data: {} to {}",
+                      last_seq_num, Header.Channel_Seq);
+                  LOG(md, Warn, "header.channel_seq {}", Header.Channel_Seq);
+                  LOG(md, Warn, "header.channel_id {}", Header.Channel_Id);
+                  LOG(md, Warn, "last_seq_number {}", last_seq_num);
+                  LOG(md, Warn, "last channel id  {}", last_channel_id);
+               }
+               last_seq_num = Header.Channel_Seq;
+               last_channel_id = Header.Channel_Id;
                handleLiveDataUnparsedMessage(
                    buffer, position, Header, listener, security_details);
                break;
@@ -514,6 +681,8 @@ bool runOnBinaryFile(
 
       start = false; // Only move leftover data on subsequent iterations
    }
+   LOG(md, Info, "Finished processing {}: last_seq_num {}, total_processed {}, total_gaps {}", 
+      file_name, last_seq_num, total_processed, gap_count);
    return true;
 }
 
@@ -575,7 +744,7 @@ void runLive(
          live_buffered_messages.clear();
          TaifexUtils::close_sockets(sockets);
          last_seq_num = 0;
-         bts_assert(reset_counter < 5);
+         bts_assert(reset_counter < 5000);
 
          state = LIVE_DATA_STATES::WAITING_FOR_REFRESH_DATA;
          break;
@@ -653,8 +822,13 @@ void runLive(
              snapshot_buffered_messages.unsafe_size());
          bts_assert(stop_refresh_thread);
 
-         while (!snapshot_buffered_messages.empty() ||
-                !products_waiting.empty()) {
+         int max_iterations = 100; // Arbitrary limit
+         int iteration_count = 0;
+
+         while ((!snapshot_buffered_messages.empty() ||
+                 !products_waiting.empty()) &&
+                iteration_count < max_iterations) {
+            iteration_count++;
             SNAPSHOT_BUFFER_MSG msg;
             if (!snapshot_buffered_messages.try_pop(msg)) {
                continue;
@@ -675,8 +849,6 @@ void runLive(
                        std::visit(
                            overloaded{
                                [&](const TaifexMessages::I010Data &data) {
-                                  // std::string  s = "looking for: '"+TaifexUtils::remove_spaces(data.prod_id)+"', in"+print(products_waiting);
-                                  //  LOG(md, Error, "{}",s);
                                   if (products_waiting.contains(
                                           TaifexUtils::remove_spaces(
                                               data.prod_id))) {
@@ -707,11 +879,26 @@ void runLive(
                     }},
                 *msg);
          }
+
+         if (iteration_count >= max_iterations) {
+            LOG(md, Error,
+                "Exceeded maximum iterations in REFRESH_DATA_DONE. Either "
+                "snapshot_buffered_messages or "
+                "products waiting are not empty.");
+            // Handle unresolved data or cleanup if necessary
+            if (!products_waiting.empty()) {
+               LOG(md, Error, "Unresolved products: {}",
+                   print(products_waiting));
+            }
+            if (!snapshot_buffered_messages.empty()) {
+               LOG(md, Error, "Unprocessed buffered messages: {}",
+                   snapshot_buffered_messages.unsafe_size());
+            }
+         }
+
          stop_reference_thread = true;
          state = LIVE_DATA_STATES::WAITING_TO_SYNC_LIVE;
          break;
-         // stop reference data
-         // apply all refresh data
       }
       case LIVE_DATA_STATES::WAITING_TO_SYNC_LIVE: {
          LOG(md, Info, "State: WAITING_TO_SYNC_LIVE");
@@ -772,10 +959,10 @@ void runLive(
             }
             if (Header.Channel_Seq != last_seq_num + 1) {
                LOG(md, Error, "Header.Channel_Seq != last_seq_num+1");
-               LOG(md, Info, "header.channel_seq {}", Header.Channel_Seq);
-               LOG(md, Info, "header.channel_id {}", Header.Channel_Id);
-               LOG(md, Info, "last_seq_number {}", last_seq_num);
-               LOG(md, Info, "last channel id  {}", last_channel_id);
+               LOG(md, Error, "header.channel_seq {}", Header.Channel_Seq);
+               LOG(md, Error, "header.channel_id {}", Header.Channel_Id);
+               LOG(md, Error, "last_seq_number {}", last_seq_num);
+               LOG(md, Error, "last channel id  {}", last_channel_id);
                state = LIVE_DATA_STATES::START;
                break;
             }
@@ -793,7 +980,57 @@ void runLive(
       };
    }
 }
-//const std::string &filename,
+
+// void bts::md::taifex::TaifexDataGateway::run(
+//    const api::md::TaifexMarketDataConfig &config,
+//    std::shared_ptr<IMarketDataListener> listner,
+//    std::shared_ptr<persistence::IDGenerator> sec_id_store) {
+//    std::shared_ptr<TaifexSecurityDetailsMapper> sec_details =
+//        std::make_shared<TaifexSecurityDetailsMapper>();
+//    if (config.read_from_file()) {
+//        bts_assert(!config.ref_data_file_name().empty());
+//        bts_assert(!config.realtime_data_file_name().empty());
+
+//        std::string base_path = config::get<bts::config::SystemConfig>().data_path();
+
+//        // Hardcode all realtime_data_20250317_* files (1 to 108)
+//        std::vector<std::string> realtime_files;
+//        for (int i = 1; i <= 108; ++i) {
+//            realtime_files.push_back(base_path + "real_time_data_20250317_" + std::to_string(i));
+//        }
+
+//        unsigned long total_files_processed = 0;
+//        unsigned long grand_total_processed = 0;
+//        int grand_total_gaps = 0;
+
+//        for (const auto& file_name : realtime_files) {
+//            LOG(md, Info, "Processing file: {}", file_name);
+//            unsigned long file_total_processed = 0;
+//            int file_gap_count = 0;
+//            bool success = runOnBinaryFile(
+//                file_name, listner, sec_id_store, sec_details, LIVE, 
+//                file_total_processed, file_gap_count);
+//            if (success) {
+//                total_files_processed++;
+//                grand_total_processed += file_total_processed;
+//                grand_total_gaps += file_gap_count;
+//            } else {
+//                LOG(md, Warn, "Failed to process file: {}", file_name);
+//            }
+//        }
+
+//        LOG(md, Info, "Processed {} files, total messages processed {}, total gaps {}", 
+//            total_files_processed, grand_total_processed, grand_total_gaps);
+
+//        while (1) {
+//            sleep_for(10s);
+//        }
+//    } else {
+//        runLive(listner, sec_id_store, sec_details);
+//    }
+// }
+// }
+// const std::string &filename,
 void bts::md::taifex::TaifexDataGateway::run(
     const api::md::TaifexMarketDataConfig &config,
     std::shared_ptr<IMarketDataListener> listner,
@@ -807,6 +1044,8 @@ void bts::md::taifex::TaifexDataGateway::run(
           config.ref_data_file_name(); //"reference_data_20240806_1";
       std::string realtime_file_name =
           config.realtime_data_file_name(); //"real_time_data_20240806_274";
+    //   std::string refresh_file_name = "refresh_data_20250317_1";
+      //  config.refresh_data_file_name();
       std::string base_path =
           config::get<bts::config::SystemConfig>().data_path();
       runOnBinaryFile(
@@ -815,6 +1054,9 @@ void bts::md::taifex::TaifexDataGateway::run(
       runOnBinaryFile(
           base_path + realtime_file_name, listner, sec_id_store, sec_details,
           LIVE);
+      // runOnBinaryFile(
+      //     base_path + refresh_file_name, listner, sec_id_store, sec_details,
+      //     REFRESH);
       while (1) { //confirm with taha
          sleep_for(10s);
       }

src/server/sparkd/md/taifex/TaifexDataProcessors.h

@@ -2,6 +2,7 @@
 #include "TaifexMessages.h"
 #include "TaifexSecurityDetailsMapper.h"
 #include "lib/core/TimeStamp.h"
+#include "lib/domain/AggressorSide.h"
 #include "lib/domain/Security.h"
 #include "lib/domain/TradeRecord.h"
 #include "lib/logging/logging.h"
@@ -9,8 +10,6 @@
 #include "lib/persistence/IDGenerator.h"
 #include "lib/utils/bts_assert.h"
 #include <cassert>
-#include "lib/config/protos/taifex_reference_data.pb.h"
-#include "lib/config/config.h"
 namespace bts::md::taifex {
 
 class TaifexDataProcessors {
@@ -72,7 +71,6 @@ class TaifexDataProcessors {
       listener->on_book(book);
       return true;
    }
-   
    static void processBookIncrements(
        const TaifexMessages::I081Data &msg,
        const std::shared_ptr<bts::md::IMarketDataListener> &listener,
@@ -85,7 +83,7 @@ class TaifexDataProcessors {
             LOG(md,Error,"sec_id or dec_loc not found for: {}",msg.prod_id);
          }
          return;
-       }
+      }
       for (const auto &entry : msg.entries) {
          IMarketDataListener::BookOp inc;
          inc.id = *sec_id;
@@ -127,19 +125,65 @@ class TaifexDataProcessors {
       for (const auto &trade_record : msg.trades) {
          auto &record = trades.emplace_back();
          record.sec_id = *sec_id;
+         // record.time_stamp = bts::core::TimeStamp::from_iso_string(
+         //     TaifexUtils::convertToISO(msg.match_time, "datetime_no_timezone"));
          record.time_stamp = bts::core::TimeStamp::from_iso_string(
              TaifexUtils::convertToISO(msg.match_time, "datetime_no_timezone"));
          record.trade_quantity = bts::domain::Quantity(trade_record.match_qty);
          record.trade_price = bts::domain::Price(
              trade_record.match_price * std::pow(10, 9 - *dec_loc));
-         // TODO: Set the aggressor side (record.agg_side)
       }
 
+      std::ostringstream oss;
+    oss << "Trades: [";
+    for (const auto &trade : trades) {
+        oss << "{Sec ID: " << trade.sec_id 
+            << ", Timestamp: " << trade.time_stamp
+            << ", Quantity: " << trade.trade_quantity.value()
+            << ", Price: " << trade.trade_price
+            << ", Aggressor Side: "
+            << ((trade.agg_side == domain::AggressorSide::Buy) ? "Buy" : "Sell")
+            << "}, ";
+    }
+    std::string trade_log = oss.str();
+    if (!trades.empty()) {
+        trade_log.pop_back(); // Remove trailing comma
+        trade_log.pop_back(); // Remove trailing space
+    }
+    trade_log += "]";
+
+    // Log the string
+    LOG(md, Diag, "{}", trade_log);
+
       std::vector<const domain::TradeRecord *> trade_ptrs;
       trade_ptrs.reserve(trades.size());
       for (const auto &trade : trades) {
          trade_ptrs.push_back(&trade);
       }
+
+      std::ostringstream ptr_oss;
+ptr_oss << "Trade Pointers: [";
+for (const auto *trade_ptr : trade_ptrs) {
+    if (trade_ptr) { // Ensure the pointer is not null
+        ptr_oss << "{Sec ID: " << trade_ptr->sec_id
+                << ", Timestamp: " << trade_ptr->time_stamp
+                << ", Quantity: " << trade_ptr->trade_quantity.value()
+                << ", Price: " << trade_ptr->trade_price
+                << ", Aggressor Side: "
+                << ((trade_ptr->agg_side == domain::AggressorSide::Buy) ? "Buy" : "Sell")
+                << "}, ";
+    }
+}
+std::string ptr_log = ptr_oss.str();
+if (!trade_ptrs.empty()) {
+    ptr_log.pop_back(); // Remove trailing comma
+    ptr_log.pop_back(); // Remove trailing space
+}
+ptr_log += "]";
+
+// Log the string
+LOG(md, Diag, "Sending to Listner {}", ptr_log);
+
       //   for (const auto *trade_ptr : trade_ptrs) {
       //          if (trade_ptr != nullptr) {
       //             LOG(md, Error, "Trade Quantity: {}, Trade Price: {}",

src/server/sparkd/md/taifex/TaifexMessages.h

@@ -3,6 +3,7 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/helper_macros.h"
 #include <bitset>
+#include <boost/algorithm/string.hpp> // Include the correct header for trim functions
 #include <cassert>
 #include <cstdint>
 #include <string>
@@ -355,24 +356,33 @@ class TaifexMessages {
          }
          // do we need this?
          size_t current_position = offset;
+
+         // Extract Product ID
          prod_id =
              TaifexUtils::extractString(buffer, current_position, PROD_ID_SIZE);
+         LOG(md, Diag, "FIND TRADE GAP: In constructor, Prod ID: {}", prod_id);
 
+         // Extract Sequence Number
          sequence_number = TaifexUtils::extractBCD_STOUL(
              buffer, current_position, SEQ_NUMBER_SIZE);
+         LOG(md, Diag, "Sequence Number: {}", sequence_number);
 
          // Extract Calculated Flag
          std::string calculated_flag_str = TaifexUtils::extractString(
              buffer, current_position, CALCULATED_FLAG);
          calculated_flag = (calculated_flag_str[0] == '1');
+         LOG(md, Diag, "Calculated Flag: {}", calculated_flag);
 
          // Extract Match Time
          match_time = TaifexUtils::time_decodePackBCD(
              buffer.data() + current_position, MATCH_TIME);
-         //match_time = TaifexUtils::convertToISO(match_time);
-         //match_time = sanitizeISOString(match_time);
-         ; // extractBCD(buffer, current_position, MATCH_TIME);
+         match_time = TaifexUtils::convertToISOFormat(match_time);
+         boost::trim(match_time);
+         LOG(md, Diag, "Match Time (ISO format): {}", match_time);
+
          current_position += MATCH_TIME;
+
+         // Extract the first trade
          trades.push_back(
              trade(buffer, current_position, FIRST_MATCH_QTY_SIZE));
          current_position += TRADE_SIZE + 2;
@@ -388,21 +398,32 @@ class TaifexMessages {
 
          auto [new_message, trades_count] =
              getFirstBitAndRemainingValue(match_display_item);
-         // assert(new_message);
+         LOG(md, Diag,
+             "Match Display Item: 0x{:02X}, New Message: {}, Trades Count: {}",
+             match_display_item, new_message, trades_count);
+
+         // Extract additional trades
          for (int i = 0; i < trades_count; i++) {
             trades.push_back(trade(buffer, current_position));
             current_position += TRADE_SIZE;
          }
-         // match quantity
+
+         // Extract Match Total Quantity
          match_total_qty = TaifexUtils::extractBCD_STRING(
              buffer, current_position, MATCH_TOTAL_QTY_SIZE);
-         // match buy count
+         LOG(md, Diag, "Match Total Quantity: {}", match_total_qty);
+
+         // Extract Match Buy Count
          match_buy_count = TaifexUtils::extractBCD_STRING(
              buffer, current_position, MATCH_BUY_COUNT_SIZE);
-         // match sell count
+         LOG(md, Diag, "Match Buy Count: {}", match_buy_count);
+
+         // Extract Match Sell Count
          match_sell_count = TaifexUtils::extractBCD_STRING(
              buffer, current_position, MATCH_SELL_COUNT_SIZE);
-         // std::cout << "trades count: " << trades_count << std::endl;
+         LOG(md, Diag, "Match Sell Count: {}", match_sell_count);
+
+         // Ensure the message ends with the expected bytes (0x0D 0x0A)
          assert(
              buffer[current_position + 1] == 0x0D &&
              buffer[current_position + 2] == 0x0A);
@@ -467,10 +488,24 @@ class TaifexMessages {
 
          // Convert month code to month number
          static int getMonthFromCode(char code) {
-            if (code >= 'A' && code <= 'L')
-               return code - 'A' +
-                      1; // 'A' corresponds to 1 (January), 'B' to 2, ..., 'L' to 12 (December)
-            throw std::runtime_error("Invalid month code");
+            if (code >= 'A' && code <= 'L') {
+               int month = code - 'A' + 1;//'A' corresponds to first January, 'B' to February etc.
+               //for logging
+               std::string str_month = std::to_string(month);
+               LOG(md, Diag, "Returned value of getMonthFromCode: {}",
+                   std::to_string(month));
+
+               return month;
+            } else {
+               //for logging
+               std::string str_code =
+                   std::string(1, code); // Convert char to string
+               LOG(md, Error,
+                   "Invalid Month Code, Returning default: 0, Received char: "
+                   "{}",
+                   str_code);
+               return 0;
+            }
          }
 
          // Method to parse Product ID and initialize ProductInfo

src/server/sparkd/md/taifex/TaifexUtils.h

@@ -270,110 +270,94 @@ class TaifexUtils {
 
       return result;
    }
+   static std::string convertToISOFormat(const std::string& input) {
+    if (input.length() != 12) {
+        throw std::invalid_argument("Input must be exactly 12 characters (hhmmssxxxxxx).");
+    }
 
-   static std::string convertToISO(
-       const std::string &match_time,
-       const std::string &format = "datetime_no_timezone") {
-      if (match_time.length() != 12) {
-         throw std::invalid_argument(
-             "in function convertToISO: match_time must be exactly 12 "
-             "characters "
-             "long: -> " +
-             match_time);
-      }
+    // Extract components from the input
+    std::string hours = input.substr(0, 2);
+    std::string minutes = input.substr(2, 2);
+    std::string seconds = input.substr(4, 2);
+    std::string microseconds = input.substr(6, 6);
 
-      try {
-         // Extract time components
-         std::string hours = match_time.substr(0, 2);
-         std::string minutes = match_time.substr(2, 2);
-         std::string seconds = match_time.substr(4, 2);
+    // Validate the time components
+    int hh = std::stoi(hours);
+    int mm = std::stoi(minutes);
+    int ss = std::stoi(seconds);
 
-         // Validate and sanitize time components
-         int h = std::abs(std::stoi(hours)) % 24;
-         int m = std::abs(std::stoi(minutes)) % 60;
-         int s = std::abs(std::stoi(seconds)) % 60;
+    if (hh < 0 || hh > 23 || mm < 0 || mm > 59 || ss < 0 || ss > 59) {
+        throw std::invalid_argument("Invalid time values in input.");
+    }
 
-         // Get the current date dynamically
-         std::string current_date = getCurrentDate();
+    // Construct the ISO 8601 time string
+    std::string isoTime = hours + ":" + minutes + ":" + seconds + "." + microseconds;
+    return isoTime;
+}
 
-         // Construct the desired ISO format
-         std::ostringstream iso_stream;
-         if (format == "datetime_with_timezone") {
-            // Full date-time with time zone: "YYYY-MM-DDTHH:MM:SS+02:00" (for simplicity using UTC "Z")
+static std::string convertToISO(const std::string &match_time, const std::string &format = "datetime_no_timezone") {
+    // Ensure match_time is not longer than 15 characters
+    if (match_time.length() > 15) {
+        throw std::invalid_argument(
+            "in function convertToISO: match_time must not be longer than 15 characters: -> " + match_time);
+    }
+
+    // If match_time is shorter than 12 characters, append '0's to the end (for microseconds)
+    std::string adjusted_match_time = match_time;
+    while (adjusted_match_time.length() < 12) {
+        adjusted_match_time += '0';  // Append '0' for microseconds
+    }
+
+    // Remove non-numeric characters (e.g., ":" and ".")
+    adjusted_match_time.erase(std::remove_if(adjusted_match_time.begin(), adjusted_match_time.end(),
+                                             [](char c) { return !std::isdigit(c); }),
+                              adjusted_match_time.end());
+
+    // Validate time components
+    try {
+        if (adjusted_match_time.length() != 12) {
+            throw std::invalid_argument("Invalid length for time format: " + match_time);
+        }
+
+        std::string hours = adjusted_match_time.substr(0, 2);
+        std::string minutes = adjusted_match_time.substr(2, 2);
+        std::string seconds = adjusted_match_time.substr(4, 2);
+        std::string microseconds = adjusted_match_time.substr(6, 6);  // Ensure this is 6 digits
+
+        // Now safely convert to integers
+        int h = std::abs(std::stoi(hours)) % 24;
+        int m = std::abs(std::stoi(minutes)) % 60;
+        int s = std::abs(std::stoi(seconds)) % 60;
+
+        // Get the current date dynamically
+        std::string current_date = getCurrentDate();
+
+        // Construct the desired ISO format
+        std::ostringstream iso_stream;
+        if (format == "datetime_with_timezone") {
             iso_stream << current_date << "T" << std::setfill('0')
                        << std::setw(2) << h << ":" << std::setfill('0')
                        << std::setw(2) << m << ":" << std::setfill('0')
-                       << std::setw(2) << s << "Z"; // Assuming UTC for now
-         } else if (format == "datetime_no_timezone") {
-            // Date-time without time zone: "YYYY-MM-DDTHH:MM:SS"
+                       << std::setw(2) << s << "." << microseconds << "Z"; // Assuming UTC for now
+        } else if (format == "datetime_no_timezone") {
             iso_stream << current_date << "T" << std::setfill('0')
                        << std::setw(2) << h << ":" << std::setfill('0')
                        << std::setw(2) << m << ":" << std::setfill('0')
-                       << std::setw(2) << s;
-         } else if (format == "date_only") {
-            // Date only: "YYYY-MM-DD"
+                       << std::setw(2) << s << "." << microseconds;
+        } else if (format == "date_only") {
             iso_stream << current_date;
-         } else {
+        } else {
             throw std::invalid_argument(
                 "Unknown format specified for convertToISO: " + format);
-         }
+        }
 
-         return iso_stream.str();
-      } catch (const std::exception &e) {
-         std::cerr << "Error in convertToISO: " << e.what() << std::endl;
-         throw;
-      }
-   }
+        return iso_stream.str();
+    } catch (const std::exception &e) {
+        std::cerr << "Error in convertToISO: " << e.what() << std::endl;
+        throw;
+    }
+}
 
-   static std::string sanitizeISOString(const std::string &isoString) {
-      std::string result = isoString;
-
-      // Remove any whitespace
-      result.erase(
-          std::remove_if(result.begin(), result.end(), ::isspace),
-          result.end());
-
-      // Ensure it's at least 20 characters long (e.g., YYYY-MM-DDTHH:MM:SS)
-      if (result.size() != 20) {
-         if (result.size() < 20) {
-            // If it's too short, pad with zeros at the end (assuming it's missing time components)
-            result.append(20 - result.size(), '0');
-         } else {
-            // If it's too long, truncate to 20 characters
-            result = result.substr(0, 20);
-         }
-      }
-
-      // Check for and correct any time zone issues or missing Z (UTC indicator)
-      size_t posT = result.find('T');
-      if (posT != std::string::npos) {
-         size_t posZ = result.find('Z', posT);
-         if (posZ == std::string::npos) {
-            size_t posMinus = result.find('-', posT);
-            size_t posPlus = result.find('+', posT);
-            if (posMinus != std::string::npos && posMinus > posT) {
-               // Check if it's "-0" or "-00" or "-000", etc.
-               if (std::all_of(
-                       result.begin() + posMinus + 1, result.end(),
-                       [](char c) { return c == '0'; })) {
-                  result = result.substr(0, posMinus) + "Z";
-               }
-            } else if (posPlus != std::string::npos && posPlus > posT) {
-               // Check if it's "+0" or "+00" or "+000", etc.
-               if (std::all_of(
-                       result.begin() + posPlus + 1, result.end(),
-                       [](char c) { return c == '0'; })) {
-                  result = result.substr(0, posPlus) + "Z";
-               }
-            } else {
-               // If there's no timezone indicator, assume UTC and add 'Z'
-               result += "Z";
-            }
-         }
-      }
-
-      return result;
-   }
 
    static std::string time_decodePackBCD(
        const unsigned char *data, size_t length) {

src/server/sparkd/md/twse/TwseDataGateway.cpp

@@ -0,0 +1,664 @@
+#include "TwseDataGateway.h"
+#include "TwseMessages.h"
+#include "lib/api/domain/SecurityType.pb.h"
+#include "lib/domain/Security.h"
+#include "lib/domain/TradeRecord.h"
+#include "lib/logging/logging.h"
+#include "lib/md/IMarketDataListener.h"
+#include "lib/persistence/IDGenerator.h"
+#include "lib/utils/helper_macros.h"
+#include <c++/11/bits/fs_fwd.h>
+#include <lib/utils/bts_assert.h>
+namespace bts::md::twse {
+
+using UNPROCESSED_DATA = std::vector<unsigned char>;
+
+bool runOnBinaryFile(
+    const std::string &file_name,
+    std::set<std::string> &securities_set,
+    const std::unordered_set<std::string> required_securities,
+    const std::string &twse_securities_path,
+    std::shared_ptr<bts::md::IMarketDataListener> &listener,
+    std::shared_ptr<persistence::IDGenerator> &sec_id_store) {
+   LOG(md, Diag, "In runOnBinaryFile");
+   LOG(md, Diag, "The path: {}", file_name);
+   LOG(md, Diag, "Absolute path: {}",
+    boost::filesystem::absolute(file_name).string());
+
+   std::ifstream file(file_name, std::ios::binary);
+   if (!file) {
+      LOG(md, Error, "File not found: {}", file_name);
+      bts_assert(false);
+   }
+
+   size_t BUFFER_SIZE = 20000;
+   [[maybe_unused]] size_t counter = 0;
+   UNPROCESSED_DATA buffer(BUFFER_SIZE);
+
+   while (file.read((reinterpret_cast<char *>(buffer.data())), BUFFER_SIZE)) {
+      size_t position = 0;
+      [[maybe_unused]] size_t format_counter = 0;
+      std::cout << std::dec << std::endl;
+      while (position < BUFFER_SIZE && buffer[position] != 0x1B) {
+         ++position;
+      }
+
+      if (position >= BUFFER_SIZE) {
+          LOG(md, Error, "Escape code (0x0B) not found in buffer");
+         break;
+      }
+
+      while (position < BUFFER_SIZE - 10) {
+         LOG(md, Diag, "Esc Code in Hexadecimal: 0x{:02X}, format counter: {}",
+         static_cast<int>(buffer[position]), format_counter);
+         std::cout << std::dec;
+         LOG(md, Diag, "Position: {}", position);
+         format_counter++;
+         position++;
+
+         auto Header = TwseMessages::Header_Fields(buffer, position);
+         std::ostringstream oss;
+         oss << Header;
+         std::string headerStr = oss.str();
+         // LOG(md, Diag, "Header: {}", headerStr);
+         //std::cout << "Header: \n" << Header;
+         if (Header.transmission_format_code == 06) {
+
+            LOG(md, Diag, "We're in Case 06");
+            auto format_6 = TwseMessages::Format_6(buffer, position);
+            std::ostringstream oss;
+            oss << format_6;
+            std::string format6Str = oss.str();
+            LOG(md, Diag, "Format 6: {}", format6Str);
+
+            
+            if(format_6.stock_code == "0050"){
+            LOG(md, Diag, "0050 book!: {}", format6Str);
+            }
+
+            domain::Book book;
+            auto &bids = book.bids();
+            auto &asks = book.asks();
+
+            bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
+                api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
+                format_6.stock_code)};
+
+            // std::vector<bts::domain::TradeRecord> trades;
+
+            book.set_security_id(sec_id);
+
+            for (unsigned long i = 0; i < format_6.bids.size(); ++i) {
+               const auto &entry = format_6.bids[i];
+
+               auto price = bts::domain::Price::from_double(entry.first / 100.0);
+               bts::domain::Quantity quantity(entry.second);
+ 
+               bids[i].price =
+                   price; //checck if the indexing starts from 1 or 0
+               bids[i].total_quantity = quantity;
+            }
+
+            // Similarly for asks
+            for (unsigned int i = 0; i < format_6.asks.size(); ++i) {
+               const auto &entry = format_6.asks[i];
+
+               auto price = bts::domain::Price::from_double(entry.first / 100.0);
+               bts::domain::Quantity quantity(entry.second);
+
+               asks[i].price = price;
+               asks[i].total_quantity = quantity;
+            }
+
+            std::vector<bts::domain::TradeRecord> trades;
+            // First, add the real-time quote if present
+            for (const auto &quote : format_6.quotes) {
+               auto &record = trades.emplace_back();
+               record.sec_id = sec_id;
+               // record.time_stamp = bts::core::TimeStamp::from_iso_string(
+               //     format_6.matching_time);
+               record.time_stamp = bts::core::TimeStamp::from_iso_string(
+                   TwseUtils::convertToISO(format_6.matching_time));
+
+               record.trade_quantity =
+                   bts::domain::Quantity(quote.volume_field);
+               record.trade_price = bts::domain::Price(
+                   stod(quote.price_field) * std::pow(10, 9 - 4));
+            }
+
+            std::vector<const domain::TradeRecord *> trade_ptrs;
+            trade_ptrs.reserve(trades.size());
+            for (const auto &trade : trades) {
+               trade_ptrs.push_back(&trade);
+            }
+
+            std::ostringstream ptr_oss;
+            ptr_oss << "Trade Pointers: [";
+            for (const auto *trade_ptr : trade_ptrs) {
+               if (trade_ptr) { // Ensure the pointer is not null
+                  ptr_oss << "{Sec ID: " << trade_ptr->sec_id
+                          << ", Timestamp: " << trade_ptr->time_stamp
+                          << ", Quantity: " << trade_ptr->trade_quantity.value()
+                          << ", Price: " << trade_ptr->trade_price
+                          << ", Aggressor Side: "
+                          << ((trade_ptr->agg_side ==
+                               domain::AggressorSide::Buy)
+                                  ? "Buy"
+                                  : "Sell")
+                          << "}, ";
+               }
+            }
+            // std::string ptr_log = ptr_oss.str();
+            if (!trade_ptrs.empty()) {
+               // ptr_log.pop_back(); // Remove trailing comma
+               // ptr_log.pop_back(); // Remove trailing space
+            }
+            // ptr_log += "]";
+
+            // Log the string
+            // LOG(md, Info, "TWSE Sending to Listner {}", ptr_log);
+
+            listener->on_book(book);
+            listener->on_trades(trade_ptrs);
+            position = position + Header.information_length - 10;
+         } else if (Header.transmission_format_code == 01) {
+            LOG(md, Diag, "We're in Case 01");
+            // Parse Format 1 message
+            auto format_1 = TwseMessages::Format_1(buffer, position);
+
+            // Convert Format_1 to string for logging
+            std::ostringstream oss;
+            oss << format_1;
+            std::string format1Str = oss.str();
+            LOG(md, Diag, "Format 1: {}", format1Str);
+
+            // Extract security ID
+            const auto id = format_1.sec_code;
+            const auto symbol = id;
+            // LOG(md, Info, "Extracted security ID: {}, Symbol: {}", id, symbol);
+
+            const std::string trimmed_id = TwseUtils::trim(id);
+            if (required_securities.find(trimmed_id) ==
+                required_securities.end()) {
+               LOG(md, Diag, "Security {} not in required list. Skipping.",
+                   trimmed_id);
+               continue;
+            }
+
+            if (securities_set.find(trimmed_id) != securities_set.end()) {
+               LOG(md, Diag, "Security {} already processed. Skipping.", id);
+            } else {
+               securities_set.insert(trimmed_id);
+
+               // Generate or retrieve SecurityID
+               bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
+                   api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
+                   id)};
+               // LOG(md, Info, "SecurityID in store: {}", sec_id);
+
+               // Populate security details
+               bts::domain::Security sec;
+               sec.security_common.venue = bts::api::domain::VENUE_TWSE;
+               sec.security_common.id = sec_id;
+               sec.security_common.exchange_sec_id = id;
+               sec.security_common.desc = id;
+               sec.security_common.symbol = symbol;
+               sec.security_common.asset = id;
+               sec.security_common.group = id;
+
+               auto get_min_inc = [](const std::string &id) {
+                  // LOG(md, Info, "Getting minimum increment for ID: {}", id);
+                  UNUSED(id);
+                  return 0.1;
+               };
+
+               // Set minimum increments
+               sec.security_common.min_price_increment = get_min_inc(id);
+               sec.security_common.min_quantity = get_min_inc(id);
+               sec.security_common.min_quantity_increment = get_min_inc(id);
+               LOG(md, Info, "Set minimum increments for security.");
+
+               sec.security_common.cash_per_point = stod(format_1.trading_unit);
+               sec.security_common.currency = format_1.trading_currency_code;
+               // LOG(md, Info, "Set cash per point: {}, Currency: {}",
+               //     sec.security_common.cash_per_point,
+               //     sec.security_common.currency);
+
+               // Serialize Security to JSON
+               nlohmann::json j;
+               to_json(j, sec);
+               std::string json_str = j.dump() + "\n";
+
+               // Write JSON to file
+               std::ofstream outfile(twse_securities_path, std::ios::app);
+               if (outfile.is_open()) {
+                  outfile << json_str;
+                  outfile.close();
+                  // LOG(md, Info, "Security data written to file.");
+               } else {
+                  // LOG(md, Info, "Failed to open securities file for writing.");
+               }
+               LOG(md, Diag, "securities_set size: {}", securities_set.size());
+
+               // Notify listener
+               std::vector<const domain::Security *> const_security_list;
+               const_security_list.push_back(&sec);
+               listener->on_securities(const_security_list);
+            }
+            // Update position
+            position = position + Header.information_length - 10;
+         } else {
+            LOG(md, Diag, "Unknown Format");
+            auto payload_length = Header.information_length;
+            position += payload_length - 10;
+         }
+      }
+      return true;
+   }
+   return false;
+}
+
+int runLive(
+    [[maybe_unused]] std::set<std::string> &securities_set,
+    [[maybe_unused]] const std::unordered_set<std::string> required_securities,
+    [[maybe_unused]] const std::string &twse_securities_path,
+    std::shared_ptr<bts::md::IMarketDataListener> &listener,
+    std::shared_ptr<persistence::IDGenerator> &sec_id_store) {
+   LOG(md, Diag, "In runLive");
+
+   //live connection
+   const std::string multicastIp = "224.0.100.100";
+   const int port = 10000;
+   const std::string interfaceIp = "10.102.167.131";
+
+   // Create a UDP socket
+   int sock = socket(AF_INET, SOCK_DGRAM, 0);
+   if (sock < 0) {
+       LOG(md, Error, "Socket creation failed");
+
+      //perror("socket creation failed");
+      return EXIT_FAILURE;
+   }
+   LOG(md, Diag, "Socket created successfully");
+
+   int reuse = 1;
+   if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) < 0) {
+      LOG(md, Error, "Failed to set SO_REUSEADDR on socket");
+
+      // perror("setsockopt(SO_REUSEADDR) failed");
+      close(sock);
+      return EXIT_FAILURE;
+   }
+
+   LOG(md, Diag, "Socket option SO_REUSEADDR set successfully");
+
+   struct sockaddr_in localAddr = {};
+   localAddr.sin_family = AF_INET;
+   localAddr.sin_port = htons(port);
+   localAddr.sin_addr.s_addr = INADDR_ANY;
+
+   if (bind(sock, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
+      LOG(md, Error, "Socket bind failed");
+
+      // perror("bind failed");
+      close(sock);
+      return EXIT_FAILURE;
+   }
+
+   LOG(md, Diag, "Socket bound successfully to port");
+
+   // Join multicast group
+   struct ip_mreq mreq;
+   mreq.imr_multiaddr.s_addr = inet_addr(multicastIp.c_str());
+   mreq.imr_interface.s_addr = inet_addr(interfaceIp.c_str());
+
+   if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) <
+       0) {
+      LOG(md, Error, "Failed to join multicast group");
+
+      // perror("setsockopt(IP_ADD_MEMBERSHIP) failed");
+      close(sock);
+      return EXIT_FAILURE;
+   }
+
+   LOG(md, Diag, "Joined multicast group successfully");
+
+   const size_t BUFFER_SIZE = 1024 * 1024;
+   std::vector<unsigned char> buffer(BUFFER_SIZE);
+
+   LOG(md, Diag, "Entering data reception loop");
+
+   while (true) {
+      struct sockaddr_in srcAddr;
+      socklen_t addrLen = sizeof(srcAddr);
+      const ssize_t sbytesRead = recvfrom(
+          sock, buffer.data(), BUFFER_SIZE, 0, (struct sockaddr *)&srcAddr,
+          &addrLen);
+      if (sbytesRead < 0) {
+         std::string err = strerror(errno);
+         LOG(md, Error, "recvfrom failed: {}", err);
+         break;
+      }
+      const size_t bytesRead = static_cast<size_t>(sbytesRead);
+      // LOG(md, Info, "Received {} bytes from {}:{}", bytesRead,
+      //  inet_ntoa(srcAddr.sin_addr), ntohs(srcAddr.sin_port));
+      // Parse received data
+      size_t position = 0;
+      size_t format_counter = 0;
+
+      while (position < bytesRead && buffer[position] != 0x1B) {
+         ++position; // Search for escape code (0x1B)
+      }
+
+      if (position >= bytesRead) {
+        // std::cerr << "Escape code (0x1B) not found in buffer" << std::endl;
+         continue;
+      }
+
+      while (position < bytesRead - 10) {
+         std::ostringstream sink;
+         sink << "Escape Code in Hexadecimal: 0x" << std::hex
+                   << std::setw(2) << std::setfill('0')
+                   << static_cast<int>(buffer[position]) << " for format #"
+                   << format_counter << std::endl;
+
+         ++format_counter;
+         ++position;
+
+         auto Header = TwseMessages::Header_Fields(buffer, position);
+         std::ostringstream oss;
+         oss << Header;
+         std::string headerStr = oss.str();
+         LOG(md, Diag, "Header: {}", headerStr);
+
+         if (Header.transmission_format_code == 6) {
+            LOG(md, Info, "Book Message --> Case 06");
+            LOG(md, Diag, "Book-->Case 06");
+            auto format_6 = TwseMessages::Format_6(buffer, position);
+            std::ostringstream oss;
+            oss << format_6;
+            std::string format6Str = oss.str();
+            LOG(md, Diag, "Format 6: {}", format6Str);
+            if(format_6.stock_code == "0050"){
+            // LOG(md, Diag, "0050 book!: {}", format6Str);
+            }
+            domain::Book book;
+            auto &bids = book.bids();
+            auto &asks = book.asks();
+
+            bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
+                api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
+                format_6.stock_code)};
+
+            // std::vector<bts::domain::TradeRecord> trades;
+
+            book.set_security_id(sec_id);
+
+            for (unsigned long i = 0; i < format_6.bids.size(); ++i) {
+               const auto &entry = format_6.bids[i];
+
+               auto price = bts::domain::Price::from_double(entry.first / 100.0);
+               bts::domain::Quantity quantity(entry.second);
+
+               bids[i].price =
+                   price; //checck if the indexing starts from 1 or 0
+               bids[i].total_quantity = quantity;
+            }
+
+            // Similarly for asks
+            for (unsigned int i = 0; i < format_6.asks.size(); ++i) {
+               const auto &entry = format_6.asks[i];
+
+               auto price = bts::domain::Price::from_double(entry.first / 100.0);
+               bts::domain::Quantity quantity(entry.second);
+
+               asks[i].price = price;
+               asks[i].total_quantity = quantity;
+            }
+
+            std::vector<bts::domain::TradeRecord> trades;
+            // First, add the real-time quote if present
+            for (const auto &quote : format_6.quotes) {
+               auto &record = trades.emplace_back();
+               record.sec_id = sec_id;
+               // record.time_stamp = bts::core::TimeStamp::from_iso_string(
+               //     format_6.matching_time);
+               record.time_stamp = bts::core::TimeStamp::from_iso_string(
+                   TwseUtils::convertToISO(format_6.matching_time));
+
+               record.trade_quantity =
+                   bts::domain::Quantity(quote.volume_field);
+               record.trade_price = bts::domain::Price(
+                   stod(quote.price_field) * std::pow(10, 9 - 4));
+            }
+
+            std::vector<const domain::TradeRecord *> trade_ptrs;
+            trade_ptrs.reserve(trades.size());
+            for (const auto &trade : trades) {
+               trade_ptrs.push_back(&trade);
+            }
+
+            std::ostringstream ptr_oss;
+            ptr_oss << "Trade Pointers: [";
+            for (const auto *trade_ptr : trade_ptrs) {
+               if (trade_ptr) { // Ensure the pointer is not null
+                  ptr_oss << "{Sec ID: " << trade_ptr->sec_id
+                          << ", Timestamp: " << trade_ptr->time_stamp
+                          << ", Quantity: " << trade_ptr->trade_quantity.value()
+                          << ", Price: " << trade_ptr->trade_price
+                          << ", Aggressor Side: "
+                          << ((trade_ptr->agg_side == domain::AggressorSide::Buy) ? "Buy" : "Sell")
+                          << "}, ";
+               }
+            }
+            std::string ptr_log = ptr_oss.str();
+            if (!trade_ptrs.empty()) {
+               ptr_log.pop_back(); // Remove trailing comma
+               ptr_log.pop_back(); // Remove trailing space
+            }
+            ptr_log += "]";
+
+            // Log the string
+            LOG(md, Diag, "TWSE Sending to Listner {}", ptr_log);
+
+            listener->on_book(book);
+            listener->on_trades(trade_ptrs);
+            position += Header.information_length - 10;
+         } else if (Header.transmission_format_code == 1) {
+            LOG(md, Info, "Case 01 --> Securities Message");
+            LOG(md, Diag, "We're in Case 01");
+            // Parse Format 1 message
+            auto format_1 = TwseMessages::Format_1(buffer, position);
+
+            // Convert Format_1 to string for logging
+            std::ostringstream oss;
+            oss << format_1;
+            std::string format1Str = oss.str();
+            LOG(md, Diag, "Format 1: {}", format1Str);
+
+            // Extract security ID
+            const auto id = format_1.sec_code;
+            const auto symbol = id;
+            LOG(md, Diag, "Extracted security ID: {}, Symbol: {}", id, symbol);
+
+            const std::string trimmed_id = TwseUtils::trim(id);
+            if (required_securities.find(trimmed_id) ==
+                required_securities.end()) {
+               LOG(md, Diag, "Security {} not in required list. Skipping.",
+                   trimmed_id);
+               continue;
+            }
+
+            if (securities_set.find(trimmed_id) != securities_set.end()) {
+               LOG(md, Diag, "Security {} already processed. Skipping.", id);
+            } else {
+               securities_set.insert(trimmed_id);
+
+               // Generate or retrieve SecurityID
+               bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
+                   api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
+                   id)};
+               // LOG(md, Info, "SecurityID in store: {}", sec_id);
+               // Populate security details
+               bts::domain::Security sec;
+               sec.security_common.venue = bts::api::domain::VENUE_TWSE;
+               sec.security_common.id = sec_id;
+               sec.security_common.exchange_sec_id = id;
+               sec.security_common.desc = id;
+               sec.security_common.symbol = symbol;
+               sec.security_common.asset = id;
+               sec.security_common.group = id;
+
+               // Helper function for minimum increment
+               auto get_min_inc = [](const std::string &id) {
+                  // LOG(md, Info, "Getting minimum increment for ID: {}", id);
+                  UNUSED(id);
+                  return 0.1;
+               };
+               // Populate other fields
+               sec.security_common.min_price_increment = get_min_inc(id);
+               sec.security_common.min_quantity = get_min_inc(id);
+               sec.security_common.min_quantity_increment = get_min_inc(id);
+               // LOG(md, Info, "Set minimum increments for security.");
+
+               sec.security_common.cash_per_point = stod(format_1.trading_unit);
+               sec.security_common.currency = format_1.trading_currency_code;
+               // LOG(md, Info, "Set cash per point: {}, Currency: {}",
+               //  sec.security_common.cash_per_point,
+               //  sec.security_common.currency);
+
+               // Serialize Security to JSON
+               nlohmann::json j;
+               to_json(j, sec);
+               std::string json_str = j.dump() + "\n";
+
+               // Write JSON to file
+               std::ofstream outfile(twse_securities_path, std::ios::app);
+               if (outfile.is_open()) {
+                  outfile << json_str;
+                  outfile.close();
+                  // LOG(md, Info, "Security data written to file.");
+               } else {
+                  // LOG(md, Info, "Failed to open securities file for writing.");
+               }
+
+               std::vector<const domain::Security *> const_security_list;
+               const_security_list.push_back(&sec);
+               listener->on_securities(const_security_list);
+            }
+            position += Header.information_length - 10;
+         } else {
+            LOG(md, Diag, "Unknown Format");
+            auto payload_length = Header.information_length;
+            position += payload_length - 10;
+         }
+      }
+   }
+   close(sock);
+   return 0;
+}
+
+void loadSecurities(
+    const std::string &filePath,
+    std::set<std::string> &securitiesSet,
+    std::shared_ptr<persistence::IDGenerator> sec_id_store,
+    std::vector<std::shared_ptr<bts::domain::Security>> &securitiesVector,
+    std::shared_ptr<IMarketDataListener> listener) {
+
+   std::ifstream infile(filePath);
+   if (!infile.is_open()) {
+      LOG(md, Error, "Failed to open securities file for reading: {}",
+       filePath);
+      return;
+   }
+
+   std::string line;
+   while (std::getline(infile, line)) {
+      try {
+         nlohmann::json j = nlohmann::json::parse(line);
+         std::shared_ptr<bts::domain::Security> sec =
+             std::make_shared<bts::domain::Security>();
+         from_json(j, *sec);
+
+         const std::string id = sec->security_common.exchange_sec_id;
+         if (securitiesSet.find(id) == securitiesSet.end()) {
+            // Generate or retrieve SecurityID
+            bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
+                api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, id)};
+            sec->security_common.id = sec_id;
+            securitiesSet.insert(id);
+            securitiesVector.push_back(sec);
+            LOG(md, Diag, "Loaded security: {}", id);
+         } else {
+            LOG(md, Diag, "Duplicate security ID: {}", id);
+         }
+      } catch (const nlohmann::json::parse_error &e) {
+         std::string err = e.what();
+         LOG(md, Error, "JSON parse error: {}", err);
+      } catch (const std::exception &e) {
+         std::string err = e.what();
+         LOG(md, Error, "Error loading security: {}", err);
+      }
+   }
+   infile.close();
+
+   // Notify listener about the loaded securities
+   if (!securitiesVector.empty()) {
+      std::vector<const bts::domain::Security *> securitiesToNotify;
+      for (const auto &sec : securitiesVector) {
+         securitiesToNotify.push_back(sec.get());
+      }
+      listener->on_securities(securitiesToNotify);
+   }
+}
+
+void bts::md::twse::TwseDataGateway::run(
+    [[maybe_unused]] const api::md::TwseMarketDataConfig &config,
+    [[maybe_unused]] std::shared_ptr<IMarketDataListener> listner,
+    [[maybe_unused]] std::shared_ptr<persistence::IDGenerator> sec_id_store) {
+
+   LOG(md, Diag, "In run, twsedatagateway.cpp");
+   std::string twse_securities_path =
+       "/home/bts/sparkdata/data/twse_load_securities.jsonl";
+   std::string sec_path = "/home/bts/sparkdata/data/1st_ip_securities_20250116_4";
+   std::string book_path =
+       "/home/bts/sparkdata/data/1st_ip_securities_20250116_27";
+   std::set<std::string> securities_set;
+   LOG(md, Diag, "securities_set size: {}", securities_set.size());
+
+   // const std::unordered_set<std::string> required_securities = {
+   //     "0050",  "2317", "2454", "2603", "2615",  "00631L", "00633L",
+   //     "00634R",  "00637L",  "00638R", "00653L",  "00654R", "00675L",  "00676R",
+   //     "00637L", "006204", "00929",  "00919",  "006208"};
+
+const std::unordered_set<std::string> required_securities = {
+    "0050",  "2317", "2454", "2603", "2615",  "00631L", "00633L",
+    "00634R",  "00637L",  "00638R", "00653L",  "00654R", "00675L",  "00676R",
+    "00637L", "006204", "00929",  "00919",  "006208",/*
+    "6239", "0050", "00703", "00706L", "00707R", "00961", "01001T", "2613",
+    "0056", "0055", "6239", "00703", "00706L", "00707R", "00961", "01001T",
+    "2613", "0056", "0055", "00657K", "00690", "00708L", "00712", "00715L",
+    "00919", "00934", "00938", "00943", "00952", "00685L",*/"054276", "054277", "054278"
+};
+
+   std::vector<std::shared_ptr<bts::domain::Security>> securities_vector;
+   loadSecurities(
+       twse_securities_path, securities_set, sec_id_store, securities_vector,
+       listner);
+   bool run_on_live = true;
+   if (run_on_live) {
+      runLive(
+          securities_set, required_securities, twse_securities_path, listner,
+          sec_id_store);
+   } else {
+      runOnBinaryFile(
+          sec_path, securities_set, required_securities, twse_securities_path,
+          listner, sec_id_store);
+
+      runOnBinaryFile(
+          book_path, securities_set, required_securities, twse_securities_path,
+          listner, sec_id_store);
+   }
+}
+
+} // namespace bts::md::twse

src/server/sparkd/md/twse/TwseDataGateway.h

@@ -0,0 +1,27 @@
+#include "lib/api/md/TwseMarketDataConfig.pb.h"
+#include <memory>  // Add this line for std::shared_ptr
+
+
+namespace bts::persistence {
+class IDGenerator;
+}
+
+namespace bts::md {
+class IMarketDataListener;
+}
+
+namespace bts::md::twse {
+
+
+class TwseDataGateway {
+public:
+    static void run(const api::md::TwseMarketDataConfig &config,
+       std::shared_ptr<IMarketDataListener> listner,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store);
+        // (void)config;
+        // (void)listener;
+        // (void)sec_id_store;
+        // // Dummy implementation
+};
+
+} // namespace bts::md::twse

src/server/sparkd/md/twse/TwseDataProcessors.h

@@ -0,0 +1,37 @@
+// TwseDataProcessors.h
+#ifndef TWSE_DATA_PROCESSORS_H
+#define TWSE_DATA_PROCESSORS_H
+
+#include "TwseMessages.h"
+#include "lib/domain/Security.h"
+#include "lib/md/IMarketDataListener.h"
+
+namespace bts::md::twse {
+class TwseDataProcessors {
+ public:
+  //  TwseMessages::Format_1 format_1;
+  //  static bool processBookMessage(
+  //      const TwseMessages::Format_6 &format_6,
+  //      const TwseMessages::Format_1 &format_1,
+  //          std::shared_ptr<bts::md::IMarketDataListener> listener) {
+      
+  //     auto sec_id = format_1.sec_code;
+  //     if (!sec_id)
+  //        return false;
+
+  //    domain::Book book;
+  //    auto &bids = book.bids();
+  //     auto &asks = book.asks();
+  //     book.set_security_id(sec_id.value());
+
+  //     for(const auto &quote : format_6.quotes){
+  //       bts::domain::Price price(quote.price_field);
+  //       bts::domain::Quantity quantity(quote.volume_field);
+  //     }
+  //     listener -> on_book(book);
+  //     return true;
+  //  }
+};
+} // namespace bts::md::twse
+
+#endif // TWSE_DATA_PROCESSORS_H

src/server/sparkd/md/twse/TwseMessages.h

@@ -0,0 +1,430 @@
+// TwseMessages.h
+#include "TwseUtils.h"
+#include "lib/logging/logging.h"
+#include <bitset>
+#include <cstdint>
+#include <ostream>
+#include <vector>
+#ifndef TWSE_MESSAGES_H
+#define TWSE_MESSAGES_H
+
+namespace bts::md::twse {
+class TwseMessages {
+ public:
+   struct Header_Fields {
+      size_t information_length;
+      size_t business_type;
+      size_t transmission_format_code;
+      size_t transmission_format_version;
+      size_t transmission_sn;
+      static const size_t HEADER_SIZE =
+          9; // Total size of header including all fields
+
+      Header_Fields(const std::vector<unsigned char> &buffer, size_t &offset) {
+         information_length =
+             stoull(TwseUtils::decodePackBCD(&buffer[offset], 2));
+         business_type =
+             stoull(TwseUtils::decodePackBCD(&buffer[offset + 2], 1));
+         transmission_format_code =
+             stoull(TwseUtils::decodePackBCD(&buffer[offset + 3], 1));
+         transmission_format_version =
+             stoull(TwseUtils::decodePackBCD(&buffer[offset + 4], 1));
+         transmission_sn =
+             stoull(TwseUtils::decodePackBCD(&buffer[offset + 5], 4));
+
+         // move offset past header
+         offset += HEADER_SIZE;
+      }
+      friend std::ostream &operator<<(
+          std::ostream &os, const Header_Fields &hf) {
+         // std::cout<<std::dec;
+         os << "Information Length: " << hf.information_length << "\n"
+            << "Business Type: " << hf.business_type << "\n"
+            << "Transmission Format Code: " << hf.transmission_format_code
+            << "\n"
+            << "Transmission Format Version: " << hf.transmission_format_version
+            << "\n"
+            << "Transmission S/N: " << hf.transmission_sn << "\n";
+         return os;
+      }
+   };
+
+   struct Format_6 {
+      std::string stock_code;
+      std::string matching_time;
+      uint8_t disclosed_item_remarks;
+      uint8_t rise_fall_remarks;
+      uint8_t status_remarks;
+      unsigned long accumalative_trading_volume;
+      std::vector<std::pair<double, uint32_t>> bids;
+      std::vector<std::pair<double, uint32_t>> asks;
+
+      struct RealTimeQuotes {
+         std::string price_field;
+         uint32_t volume_field;
+
+         RealTimeQuotes(
+             const std::vector<unsigned char> &buffer,
+             const size_t &offset = 0) {
+
+            price_field = TwseUtils::decodePackBCD(&buffer[offset], 5);
+            volume_field =
+                stoul(TwseUtils::decodePackBCD(&buffer[offset + 5], 4));
+            //offset = offset + 9;
+         }
+
+         friend std::ostream &operator<<(
+             std::ostream &os,
+             const Format_6::RealTimeQuotes &real_time_quotes) {
+            os << "\n Price Field: " << real_time_quotes.price_field << "\n"
+               << "Volume Field: " << real_time_quotes.volume_field << "\n";
+            return os;
+         }
+      };
+      std::vector<RealTimeQuotes> quotes;
+      // std::vector<RealTimeQuotes> quotes;
+      friend std::ostream &operator<<(
+          std::ostream &os, const Format_6 &format) {
+         os << "\nStock Code: ." << format.stock_code << ".\n"
+            << "Matching Time: " << format.matching_time << "\n"
+            << "Disclosed Item Remarks (Binary): "
+            << std::bitset<8>(format.disclosed_item_remarks)
+            << " (Decimal: " << static_cast<int>(format.disclosed_item_remarks)
+            << ")\n"
+            << "Rise/Fall Remarks (Binary): "
+            << std::bitset<8>(format.rise_fall_remarks)
+            << " (Decimal: " << static_cast<int>(format.rise_fall_remarks)
+            << ")\n"
+            << "Status Remarks (Binary): "
+            << std::bitset<8>(format.status_remarks)
+            << " (Decimal: " << static_cast<int>(format.status_remarks) << ")\n"
+            << "Accumulative Trading Volume: "
+            << format.accumalative_trading_volume << "\n";
+
+         os << "Real Time Quotes:\n";
+         for (const auto &quote : format.quotes) {
+            os << quote;
+         }
+         os << "Bids: \n";
+         for (const auto &bid : format.bids) {
+            os << "Price = " << bid.first << " Quantity = " << bid.second
+               << "\n";
+         }
+
+         os << "\nAsks: \n";
+         for (const auto &ask : format.asks) {
+            os << "Price = " << ask.first << " Quantity = " << ask.second
+               << "\n";
+         }
+         // os << "Quotes:\n";
+         //  for (const auto &real_time_quotes : format.quotes) {
+         //     os << real_time_quotes;
+         //  }
+         return os;
+      }
+
+      Format_6(
+          const std::vector<unsigned char> &buffer, const size_t &offset = 0) {
+         size_t position = offset;
+
+         stock_code = std::string(
+             buffer.begin() + position, buffer.begin() + position + 6);
+         stock_code = TwseUtils::trim(stock_code);
+         position += 6;
+
+         matching_time = TwseUtils::decodePackBCD(&buffer[position], 6);
+         position += 6;
+
+         disclosed_item_remarks = buffer[position];
+         position += 1;
+
+         rise_fall_remarks = buffer[position];
+         position += 1;
+
+         status_remarks = buffer[position];
+         position += 1;
+
+         accumalative_trading_volume =
+             stoul(TwseUtils::decodePackBCD(&buffer[position], 4));
+         position += 4;
+
+         if (disclosed_item_remarks & 0x80) { // If Bit 7 is 1
+            LOG(md, Diag, "Price Field and Volume Field present");
+            // std::cin.ignore();
+            auto rtq = RealTimeQuotes(buffer, position);
+            quotes.push_back(rtq);
+         } else {
+            LOG(md, Diag, "No Price Field and Volume Field");
+         }
+         LOG(md, Diag, "Purchasing Price and Volumes");
+
+         std::cout << std::dec;
+         // Read Purchase Price and Volume
+         uint8_t purchasing_count =
+             (disclosed_item_remarks >> 4) & 0x07; // Get bits 6-4
+         for (uint8_t i = 0; i < purchasing_count; ++i) {
+            std::string purchase_price =
+                TwseUtils::decodePackBCD(&buffer[position], 5); // Price Field
+            double bid_price =
+                std::stod(TwseUtils::decodePackBCD(&buffer[position], 5));
+            position += 5; // Move to the volume field
+
+            std::string volume_field =
+                TwseUtils::decodePackBCD(&buffer[position], 4); // Volume Field
+            uint32_t bid_volume =
+                stoul(TwseUtils::decodePackBCD(&buffer[position], 4));
+            position += 4; // Move to the next price field
+            LOG(md, Info, "Purchase Price {}: {} Volume {}: {}", i + 1,
+                purchase_price, i + 1, volume_field);
+            bids.emplace_back(bid_price, bid_volume);
+         }
+
+         // Read Sell Price and Volume if applicable
+         uint8_t selling_count =
+             (disclosed_item_remarks >> 1) & 0x07; // Get bits 3-1
+         for (uint8_t i = 0; i < selling_count; ++i) {
+            std::string sell_price = TwseUtils::decodePackBCD(
+                &buffer[position], 5); // Price Field: 5 bytes
+            double ask_price =
+                std::stod(sell_price); // Use the 5-byte decoded value
+            position += 5;             // Move to the volume field
+
+            std::string volume_field = TwseUtils::decodePackBCD(
+                &buffer[position], 4); // Volume Field: 4 bytes
+            uint32_t ask_volume =
+                stoul(volume_field); // Use the 4-byte decoded value
+            position += 4;           // Move to the next price field
+
+            LOG(md, Info, "Sell Price {}: {} Volume {}: {}", i + 1, sell_price,
+                i + 1, volume_field);
+
+            asks.emplace_back(ask_price, ask_volume);
+         }
+      }
+   };
+
+   struct Format_1 {
+      std::string sec_code;
+      std::string stock_abbreviation;
+      std::string industry_category;
+      std::string stock_category; //can be blank
+      std::string stock_entries;  //can be blank
+      std::string stock_anomaly_code;
+      std::string board_code;
+      std::string today_currency_price;
+      std::string trading_currency_code;
+      std::string rise_stop_price;
+      std::string fall_stop_price;
+      std::string non_10_face_value_indicator;
+      std::string abnormal_recommendation_indicator;
+      std::string abnormal_securities_indicator;
+      std::string day_trading_indicator;
+      std::string exemption_unchanged_market_margin_sale_indicator;
+      std::string exemption_unchanged_market_securities_lending_sale_indicator;
+      std::string matching_cycle_seconds;
+      std::string warrant_id;
+      std::string
+          exercise_strike_price; // This will need to be decoded from PACK BCD
+      std::string
+          prev_business_day_exercise_volume; // This will also need PACK BCD decoding
+      std::string prev_business_day_cancellation_volume; // PACK BCD format
+      std::string issuing_balance_volume;                // PACK BCD format
+      std::string strike_ratio;                          // PACK BCD format
+      std::string upper_limit_price;                     // PACK BCD format
+      std::string lower_limit_price;                     // PACK BCD format
+      std::string maturity_date;
+      std::string foreign_stock_id;
+      std::string trading_unit; // PACK BCD format
+      std::string trading_currency_code_foreign_stock;
+      std::string market_information_note;
+
+      Format_1(
+          const std::vector<unsigned char> &buffer, const size_t &offset = 0) {
+         size_t position = offset;
+
+         sec_code = std::string(
+             buffer.begin() + position, buffer.begin() + position + 6);
+         sec_code = TwseUtils::trim(sec_code);
+         position += 6;
+
+         stock_abbreviation = std::string(
+             buffer.begin() + position, buffer.begin() + position + 16);
+         position += 16;
+
+         industry_category = std::string(
+             buffer.begin() + position, buffer.begin() + position + 2);
+
+         //                                    std::cout << "Industry Category (Hex): ";
+         // for (char c : industry_category_str) {
+         //     std::cout << std::setw(2) << std::setfill('0') << std::hex <<
+         //     (int)(unsigned char)c << " ";
+         // }
+         // std::cout << std::endl;
+         // Convert the ASCII string to an integer
+         position += 2;
+
+         //hex values show pointing at the right place
+         stock_category = std::string(
+             buffer.begin() + position, buffer.begin() + position + 2);
+         //         std::cout << "Stock Category (String): " << stock_category << std::endl;
+         // std::cout << "Stock Category (ASCII Values): ";
+         // for (char c : stock_category) {
+         //     std::cout << (int)(unsigned char)c << " ";  // Prints the ASCII value of each character
+         // }
+         // std::cout << std::endl;
+
+         // std::cout << "Stock Category (Hex): ";
+         // for (char c : stock_category) {
+         //     std::cout << std::setw(2) << std::setfill('0') << std::hex <<
+         //     (int)(unsigned char)c << " ";
+         // }
+         // std::cout << std::endl;
+         position += 2;
+
+         stock_entries = std::string(
+             buffer.begin() + position, buffer.begin() + position + 2);
+         position += 2;
+
+         stock_anomaly_code = TwseUtils::decodePackBCD(&buffer[position], 1);
+         position += 1;
+
+         board_code = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         today_currency_price = TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         rise_stop_price = TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         fall_stop_price = TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         non_10_face_value_indicator = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         abnormal_recommendation_indicator = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         abnormal_securities_indicator = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         day_trading_indicator = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         exemption_unchanged_market_margin_sale_indicator = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         exemption_unchanged_market_securities_lending_sale_indicator =
+             std::string(
+                 buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         matching_cycle_seconds =
+             TwseUtils::decodePackBCD(&buffer[position], 3);
+         position += 3;
+
+         warrant_id = std::string(
+             buffer.begin() + position, buffer.begin() + position + 1);
+         position += 1;
+
+         exercise_strike_price = TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         prev_business_day_exercise_volume =
+             TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         prev_business_day_cancellation_volume =
+             TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         issuing_balance_volume =
+             TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         strike_ratio = TwseUtils::decodePackBCD(&buffer[position], 4);
+         position += 4;
+
+         upper_limit_price = TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         lower_limit_price = TwseUtils::decodePackBCD(&buffer[position], 5);
+         position += 5;
+
+         maturity_date = TwseUtils::decodePackBCD(&buffer[position], 4);
+         position += 4;
+
+         foreign_stock_id = TwseUtils::decodePackBCD(&buffer[position], 1);
+         position += 1;
+
+         trading_unit = TwseUtils::decodePackBCD(&buffer[position], 3);
+         position += 3;
+
+         trading_currency_code = TwseUtils::decodePackBCD(&buffer[position], 3);
+         position += 3;
+
+         market_information_note =
+             TwseUtils::decodePackBCD(&buffer[position], 1);
+         position += 1;
+      }
+
+      friend std::ostream &operator<<(
+          std::ostream &os, const Format_1 &format) {
+         os << "\nSEC Code: " << format.sec_code << "\n"
+            << "Stock Abbreviation: " << format.stock_abbreviation << "\n"
+            << "Industry Category: " << format.industry_category << "\n"
+            << "Stock Category(can be blank): " << format.stock_category << "\n"
+            << "Stock Entries(can be blank): " << format.stock_entries << "\n"
+            << "Stock Anomaly code: " << format.stock_anomaly_code << "\n"
+            << "Board Code: " << format.board_code << "\n"
+            << "Today Currency Price: " << format.today_currency_price << "\n"
+            << "Rise Stop Price: " << format.rise_stop_price << "\n"
+            << "Fall Stop Price: " << format.fall_stop_price << "\n"
+            << "Non-$10 Face Value Indicator: "
+            << format.non_10_face_value_indicator << "\n"
+            << "Abnormal Recommendation Indicator: "
+            << format.abnormal_recommendation_indicator << "\n"
+            << "Abnormal Securities Indicator: "
+            << format.abnormal_securities_indicator << "\n"
+            << "Day Trading Indicator: " << format.day_trading_indicator << "\n"
+            << "Exemption of Unchanged Market Margin Sale Indicator: "
+            << format.exemption_unchanged_market_margin_sale_indicator << "\n"
+            << "Exemption of Unchanged Market Securities Lending Sale "
+               "Indicator: "
+            << format
+                   .exemption_unchanged_market_securities_lending_sale_indicator
+            << "\n"
+            << "Matching Cycle Seconds: " << format.matching_cycle_seconds
+            << "\n"
+            << "Warrant ID: " << format.warrant_id << "\n"
+            << "Exercise (Strike) Price: " << format.exercise_strike_price
+            << "\n"
+            << "Previous Business Day Exercise Volume: "
+            << format.prev_business_day_exercise_volume << "\n"
+            << "Previous Business Day Cancellation Volume: "
+            << format.prev_business_day_cancellation_volume << "\n"
+            << "Issuing Balance Volume: " << format.issuing_balance_volume
+            << "\n"
+            << "Strike Ratio: " << format.strike_ratio << "\n"
+            << "Upper Limit Price: " << format.upper_limit_price << "\n"
+            << "Lower Limit Price: " << format.lower_limit_price << "\n"
+            << "Maturity Date: " << format.maturity_date << "\n"
+            << "Foreign Stock ID: " << format.foreign_stock_id << "\n"
+            << "Trading Unit: " << format.trading_unit << "\n"
+            << "Trading Currency Code: " << format.trading_currency_code << "\n"
+            << "Market Information Note: " << format.market_information_note
+            << "\n";
+         return os;
+      }
+   };
+};
+} //namespace bts::md::twse
+
+#endif // TWSE_MESSAGES_H

src/server/sparkd/md/twse/TwseSecurityDetailsMapper.h

@@ -0,0 +1,13 @@
+// TwseSecurityDetailsMapper.h
+#ifndef TWSE_SECURITY_DETAILS_MAPPER_H
+#define TWSE_SECURITY_DETAILS_MAPPER_H
+
+namespace TWSE {
+    class TwseSecurityDetailsMapper {
+    public:
+        TwseSecurityDetailsMapper() = default;
+        ~TwseSecurityDetailsMapper() = default;
+    };
+}
+
+#endif // TWSE_SECURITY_DETAILS_MAPPER_H