fix build: Upgrade IronPython to 3.4.2

All csprojs need to be on same version (Spark.csproj was on 3.4.1, while
DataStream.csproj was on 3.4.2)

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -0,0 +1,49 @@
+using System;
+
+namespace BCG {
+   [DataStreamUserRespresentation("count")]
+   public class CountStream : ViewStream {
+      private DataPacket myLastFiredDataPacket;
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
+      private int myId = 0;
+      private ViewStream myVS;
+
+      public CountStream(ViewStream vs) {
+         myVS = vs;
+      }
+
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
+
+      protected override void myOnFirstSubscriber() {
+         myVS.onChange += new ChangeDelegate(myHandleChanges);
+      }
+      protected override void myOnLastSubscriber() {
+         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+      }
+
+
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
+            myLastFiredDataPacket =
+               myLastFiredDataPacket == null ?
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
+                  :
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  myLastFiredDataPacket.newRows)
+            );
+      }
+
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
+         retVal.addRow
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
+         return retVal;
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/DateTimeExpanderStream.cs

@@ -0,0 +1,38 @@
+namespace bts.BlueOrb {
+   using bts;
+
+   using BCG;
+
+   using System;
+   [DataStreamUserRespresentation("DateTimeExpanderStream")]
+   public class DateTimeExpanstionStream : ViewStream {
+      public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
+            IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
+            rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day_of_week");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_minute");
+            rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
+            return rv;
+         };
+         ViewStream rs = new MapStream(parent_stream,
+         s => get_schema(s),
+         r => {
+            DateTime dt = r.get_datetime(inp_col);
+            IBlueTableSchema s = get_schema(r.schema);
+            MutableBlueRow rv = new MutableBlueRow(s, r);
+            rv.setValue($"{inp_col}_date", dt.Date);
+            rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
+            rv.setValue($"{inp_col}_day_of_month", dt.Day);
+            rv.setValue($"{inp_col}_month", dt.Date.Month);
+            rv.setValue($"{inp_col}_hour", dt.Hour);
+            rv.setValue($"{inp_col}_minute", dt.Minute);
+            rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
+            return rv;
+         });
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -0,0 +1,158 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System;
+   using System.Text.RegularExpressions;
+
+
+   public class RegExpFilter : IStreamFilter {
+      private Regex[] myRegExps;
+      private string[] myCols;
+
+      public RegExpFilter(string col, string regExp)
+         : this(new string[] { col }, new string[] { regExp }) { }
+
+      public RegExpFilter(string[] cols, string[] regExps) {
+         myCols = cols;
+         myRegExps = new Regex[regExps.Length];
+         for (int i = 0; i < regExps.Length; i++)
+            myRegExps[i] = new Regex(regExps[i]);
+      }
+
+      #region IStreamFilter Members
+      public bool filter(IBlueRow dr) {
+         bool rv = true;
+         for (int i = 0; i < myCols.Length; i++) {
+            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+            if (!rv)
+               return rv;
+         }
+         return rv;
+      }
+      #endregion
+   }
+
+   //[DataStreamUserReprsentation("FilterStream")]
+   public class FilterStream : ViewStream {
+      private BlueTable myLastFiredRows;
+      private IBlueRowCollection myLastReceivedRows;
+      private IStreamFilter myFilter;
+      private ViewStream myParentStream;
+      private string myFilterString;
+      private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
+
+      #region constructors
+      public delegate bool OnFilter(IBlueRow r);
+      private class OnFilterWrapper : IStreamFilter {
+         public OnFilterWrapper(OnFilter f) { this.f = f; }
+
+         public bool filter(IBlueRow dr) { return f(dr); }
+
+         private OnFilter f;
+      }
+      public FilterStream(ViewStream vs, OnFilter f)
+         : this(vs, new OnFilterWrapper(f)) { }
+
+
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
+         {
+            myParentStream = vs;
+            myFilter = filter;
+            myFilterString = filter.ToString();
+            if (filter is IDynamicStreamFilter)
+               (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
+         }
+      }
+
+      public FilterStream(ViewStream vs, string filterExpression)
+         : this(vs, new BooleanExpressionFilter(filterExpression)) {
+         myFilterString = filterExpression;
+      }
+
+      public FilterStream(ViewStream vs, string col, string regExp)
+         : this(vs, new RegExpFilter(col, regExp)) {
+         myFilterString = col + " , " + regExp;
+      }
+
+      public FilterStream(ViewStream vs)
+         : this(vs, new BinaryFilter(true)) {
+         myFilterString = true.ToString();
+      }
+      #endregion
+
+      #region overridden methods
+      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+
+      protected override void myOnLastSubscriber() {
+         myLastFiredRows = null;
+         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+      }
+
+      protected override void myFireEvent(object o) {
+         DataPacket dp = o as DataPacket;
+         if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
+            base.myFireEvent(o);
+      }
+      #endregion
+
+      private void myHandleCallbacks(object newValue) {
+         DataPacket retVal;
+         DataPacket arrivedPacket = newValue as DataPacket;
+         myLastReceivedRows = arrivedPacket.newRows;
+         if (myLastFiredRows != null) {
+            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+               new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.deletedRows)
+               if (myLastFiredRows.containsID((int)r["id"])) {
+                  myLastFiredRows.deleteRow(r);
+                  (retVal.deletedRows as BlueTable).addRow(r);
+               }
+            foreach (IBlueRow r in arrivedPacket.addedRows)
+               if (myFilter.filter(r)) {
+                  myLastFiredRows.addRow(r);
+                  (retVal.addedRows as BlueTable).addRow(r);
+               }
+         } else {
+            myLastFiredRows = new BlueTable(arrivedPacket.schema);
+            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+            foreach (IBlueRow r in arrivedPacket.newRows)
+               if (myFilter.filter(r))
+                  myLastFiredRows.addRow(r);
+         }
+         myFireEvent(retVal);
+      }
+
+      public IStreamFilter filter {
+         set {
+            {
+               myFilter = value;
+               if (numberOfSubscribers > 0) {
+                  if (myLastReceivedRows != null) {
+                     BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
+                     foreach (IBlueRow r in myLastReceivedRows)
+                        if (myFilter.filter(r))
+                           nRows.addRow(r);
+                     IBlueRowCollection dRows = new BlueTable(nRows.schema),
+                        aRows = new BlueTable(nRows.schema);
+                     BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
+                     myLastFiredRows = nRows;
+                     myFireEvent(new DataPacket(nRows, aRows, dRows));
+                  }
+               }
+            }
+         }
+         get {            
+               return myFilter;
+         }
+      }
+
+      //public string filterString { set { filter = new BooleanExpressionFilter(value); } }
+
+      public override string ToString() {
+         return GetType().ToString() + " " + myFilterString;
+      }
+
+      private void myHandleFilterChange() {
+         filter = myFilter;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text.Json;
+
+using bts;
+//using bts.SilverPine;
+using BCG;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("GroupByStream")]
+   public class GroupByStream : ViewStream {
+
+      public interface INonIncGroupByEvaluator {
+         // grouped_rows
+         IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
+         string[] get_group_by_cols();
+      }
+
+      private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+         Func<string[]> _groupby_cols_func;
+         Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
+         public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
+                                          Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
+            _groupby_cols_func = groupby_cols_func;
+            _evaluate_func = evaluate_func;
+         }
+         public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+            BlueTable rv = null;
+            foreach (var kv in grouped_rows) {
+               List<IBlueRow> g_rows = kv.Value;
+               List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
+               if (out_rows != null && out_rows.Count > 0) {
+                  rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+               }
+               rv.addRows(out_rows);
+            }
+            return rv;
+         }
+         public string[] get_group_by_cols() {
+            return _groupby_cols_func();
+         }
+
+      }
+      private ViewStream _main_stream;
+      private INonIncGroupByEvaluator _groupby_func;
+      private BlueRowHasher _row_hasher;
+      private string[] _groupby_cols;
+      private Stopwatch _timer = new();
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
+         _main_stream = main_stream;
+         _groupby_func = groupby_func;
+         _groupby_cols = _groupby_func.get_group_by_cols();
+         _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
+      }
+
+      public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
+                                 Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
+                                 : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text)
+      : this(parent_str, groupby_col, standard_group_by_text, String.Empty) { }
+
+      // this is the main constructor that does everything
+      public GroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_ops, string optional_sort_col) {
+         MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
+         Func<string[]> groupby_cols_func = () => {
+            return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
+         };
+         Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
+            if (!String.IsNullOrEmpty(optional_sort_col))
+               group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
+            // this si where the group by aggregation happens
+            IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
+            return new IBlueRow[] { rv }.ToList();
+         };
+         ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
+         mySetRealSource(rs);
+      }
+      public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
+      this(parent_str, String.Empty, standard_group_by_text) {
+      }
+
+
+      protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
+
+      protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
+
+      private void _main_stream_onchange(object nv) {
+         DataPacket dp = nv as DataPacket;
+         ///grouping together
+         _timer.Restart();
+         IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+         new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
+         BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         /// reducing
+         _timer.Restart();
+         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _n_total_fires++;
+         if (applyElapsedMs >= 50) {
+            _n_expensive_fires++;
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         // TODO,
+         if (total_state != null)
+            myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+
+namespace bts.BlueOrb {   
+   public interface INonIncJoinInlineMapper
+   {      
+      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
+
+      IBlueRowCollection map_rows(IBlueRowCollection rows);
+      
+      event MapperChangeDelegate onMapperChange;
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs

@@ -0,0 +1,152 @@
+using bts;
+using bts.collections;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography;
+using BCG;
+
+namespace bts.BlueOrb {   
+   internal class NonIncJoinBookkeeper : IDisposable {
+      public Action on_new_data;
+      private readonly NonIncJoinSpec _join_spec;
+      private readonly int _join_position;
+      private readonly int _start_index_in_output;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly bool _is_left_most;
+      private readonly bool _is_tag_along;
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+
+      private IBlueTableSchema _schema = null;
+      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
+      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
+      private IBlueRow _solitary_row = null;
+
+      private int[] _join_col_indices = null,
+                     _data_col_indices = null;
+
+      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
+         _join_spec = the_join_spec;
+         _join_position = join_position;
+         _start_index_in_output = start_index_in_output;
+         _row_hasher = row_hasher;
+         _is_left_most = _join_position == 0;
+         _is_tag_along = _join_spec.join_col_names.Length == 0;
+      }
+
+      private bool _filled_missing_data = false;
+      private void _fill_missing_data(DataPacket dp) {
+         if (_filled_missing_data)
+            return;
+         Func<string[], bool> null_or_empty = (str_arr) => {
+            if (str_arr == null || str_arr.Length == 0)
+               return true;
+            foreach (string s in str_arr) {
+               if (!String.IsNullOrEmpty(s)) {
+                  return false;
+               }
+            }
+            return true;
+         };
+         Func<string[], string[]> remove_bad_cols = (col_names) => {
+            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
+            List<string> cols_list = new List<string>(col_names);
+            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
+            return cols_list.ToArray();
+         };
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
+         if (null_or_empty(_join_spec.data_col_names)) {
+            _join_spec.data_col_names = dp.schema.columnNames;
+         }
+         if (null_or_empty(_join_spec.projected_col_names)) {
+            _join_spec.projected_col_names = _join_spec.data_col_names;
+         }
+      }
+
+      public void init() {
+         _join_spec.stream.onChange += _on_stream_cb;
+      }
+
+      public void _on_stream_cb(object dp_obj) {
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         var timer = Stopwatch.StartNew();
+         DataPacket dp = (DataPacket)dp_obj;
+         _fill_missing_data(dp);
+         if (_schema == null) {
+            _schema = dp.schema;
+            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
+            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
+         }
+
+         if (_is_left_most) {
+            _row_list.Clear();
+            foreach (IBlueRow r in dp.newRows)
+               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
+         } else {
+            if (_is_tag_along)
+               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
+            else {
+               _row_map.Clear();
+               foreach (IBlueRow r in dp.newRows)
+                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
+            }
+         }
+         timer.Stop();
+         _n_total_fires++;
+         if (timer.ElapsedMilliseconds >= 50) {
+            _n_expensive_fires++;
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
+         }
+         if (on_new_data != null)
+            on_new_data();
+      }
+
+      public bool is_ready { get { return _schema != null; } }
+
+      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
+         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
+            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
+      }
+
+      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
+
+      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
+         IBlueRow this_row = _row_list[row_index].second;
+         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+      }
+
+      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+         if (_is_tag_along) {
+            if (_solitary_row != null)
+               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         } else {
+            IBlueRow this_row = null;
+            if (_row_map.TryGetValue(row_hash, out this_row))
+               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
+            else
+               copy_dbnull_to_output_row(output_row);
+         }
+      }
+
+      private void copy_dbnull_to_output_row(object[] output_row) {
+         for (int i = 0; i < _data_col_indices.Length; i++)
+            output_row[i + _start_index_in_output] = DBNull.Value;
+      }
+
+      #region IDisposable
+
+      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using BCG;
+
+using mbts;
+
+namespace bts.BlueOrb {   
+   // this a collection of spec, theres no joining logic in here
+   public class NonIncJoinSpec {
+      public ViewStream stream;
+      // join col names are the names on which the streams are joined (for equality),
+      // data col names are the names in the original stream
+      // projected col names are their new names, ie if its x:l_x, then x is in the 
+      // data_col_names and l_x in projected cols
+      public string[] join_col_names, data_col_names, projected_col_names;
+
+      public NonIncJoinSpec() { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
+         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
+
+      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
+         stream = the_stream;
+         join_col_names = the_join_col_names;
+         data_col_names = the_data_col_names;
+         projected_col_names = the_projected_col_names;
+      }
+
+      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
+         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
+         foreach (Triplet<ViewStream, string, string> r in join_specs) {
+            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
+            string[] visible_cols = r.third.Split(' ');
+            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
+            foreach (string s in visible_cols) {
+               string[] toks = s.Split(':');
+               if (toks.Length == 1) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[0]);
+               } else if (toks.Length == 2) {
+                  data_col_names.Add(toks[0]);
+                  mapped_col_names.Add(toks[1]);
+               }
+            }
+            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
+         }
+         return rv.ToArray();
+      }
+
+      public override string ToString() {
+         Func<string[], string> arr_txt = (a) => {
+            string rv = "";
+            foreach (string s in a) {
+               rv += "{s} ";
+            }
+            rv = rv.Trim();
+            return rv;
+         };
+         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs

@@ -0,0 +1,165 @@
+using bts;
+
+using mbts;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using BCG;
+
+namespace bts.BlueOrb {   
+   [DataStreamUserRespresentation("JoinStream")]
+   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
+      private readonly NonIncJoinSpec[] _join_specs;
+      private readonly INonIncJoinInlineMapper _output_mapper;
+      private readonly BlueRowHasher _row_hasher;
+      private readonly TimeSpan _throttle_period;
+      private readonly ViewStreamDatabase _vsd;
+
+      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
+      private IBlueTableSchema _output_schema = null;
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private bool _is_dirty = false;
+
+      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
+                              ViewStream right, string right_join_cols, string right_proj)
+         : this(new Triplet<ViewStream, string, string>[]
+               {
+                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
+                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
+               }) { }
+
+      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
+                              params Triplet<ViewStream, string, string>[] join_specs)
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
+      }
+
+      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
+         _join_specs = join_specs;
+         _output_mapper = output_mapper;
+         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
+         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
+         _vsd = vsd;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         int data_col_no = 1;
+         for (int jp = 0; jp < _join_specs.Length; jp++) {
+            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            data_col_no += _join_specs[jp].data_col_names.Length;
+            s.on_new_data += _stream_callback;
+            _book_keeping_data.Add(s);
+         }
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange += _on_mapper_change;
+         _vsd.addWorker(this);
+         foreach (var bkd in _book_keeping_data)
+            bkd.init();
+      }
+
+      protected override void myOnLastSubscriber() {
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            d.Dispose();
+         if (_output_mapper != null)
+            _output_mapper.onMapperChange -= _on_mapper_change;
+         _vsd.removeWorker(this);
+      }
+
+      private void _stream_callback() {
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         _is_dirty = true;
+         doWork();
+      }
+
+      private void _on_mapper_change() {
+         _is_dirty = true;
+         doWork();
+      }
+
+      private bool _is_ready() {
+         bool rv = true;
+         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+            rv &= d.is_ready;
+         return rv;
+      }
+
+      private void _construct_schema() {
+         if (_output_schema == null) {
+            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
+            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
+               d.constribute_cols_to_schema(cis);
+            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
+         }
+      }
+
+      private int _n_total_fires = 0, _n_expensive_fires = 0;
+      private void _form_and_fire() {
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         if (_is_ready()) {
+            _construct_schema();
+            var timer = Stopwatch.StartNew();
+            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
+            List<object[]> output_rows = new List<object[]>();
+
+            for (int i = 0; i < left_row_list.Count; i++) {
+               object[] or = new object[_output_schema.columnInfo.Length];
+               or[0] = i;
+               output_rows.Add(or);
+            }
+
+            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
+               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
+               for (int i = 0; i < left_row_list.Count; i++) {
+                  if (ji == 0)
+                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
+                  else
+                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
+               }
+            }
+            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
+            timer.Stop();
+            _n_total_fires++;
+            //if (timer.ElapsedMilliseconds >= 250) {
+            _n_expensive_fires++;
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
+            //}
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
+         }
+      }
+
+      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
+         if (_output_mapper != null) {
+            IBlueRowCollection output_rows;
+            var timer = Stopwatch.StartNew();
+            output_rows = _output_mapper.map_rows(unmapped_rows);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
+            }
+            return output_rows;
+         } else {
+            return unmapped_rows;
+         }
+      }
+
+      public void doWork() {
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
+            _is_dirty = false;
+            _last_fire_time = DateTime.Now;
+            _form_and_fire();
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/JsonSaverStream.cs

@@ -0,0 +1,28 @@
+//namespace bts.Blueorb {
+//   using BCG;
+
+//   public class JsonSaverStream : ViewStream {
+//      private ViewStream _stream;
+//      private string _file_name;
+//      public JsonSaverStream(ViewStream vs, string file_name) {
+//         _stream = vs;
+//         _file_name = file_name;
+//      }
+
+//      protected override void myOnFirstSubscriber() {
+//         _stream.onChange += _on_data;
+//      }
+
+//      private void _on_data(object dp_obj) {
+//         if (dp_obj == null) {
+//            return;
+//         }
+//         DataPacket dp = dp_obj as DataPacket;
+//         dp.newRows.write_to_json_file(dp.newRows, _file_name);
+//      }
+
+//      protected override void myOnLastSubscriber() {
+//         _stream.onChange -= _on_data;
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/BlueOrb/LiteralStream.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+using Amazon.Runtime.Internal.Util;
+
+using BCG;
+using BCG.Utilities;
+
+using bts;
+
+namespace bts.BlueOrb{
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
+      // col_name: col_value ... ; col_name: col_value
+      public LiteralStream(string table_name, string rows) {
+         Func<string, int, IBlueRow> get_row = (json_row, id) => {
+            Dictionary<string, object> map = new Dictionary<string, object>();
+            string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
+            //Logger.assert(col_vals.Length == 2);
+            foreach (string t in toks) {
+               var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
+               string col = col_vals[0];
+               string val = col_vals[1];
+               map.Add(col, val);
+            }
+            map["id"] = id;
+            return new BlueRow(map);
+         };
+         string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
+         BlueTable table = null;
+         foreach (string json_string in row_arr) {
+            IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
+            get_row(json_string, table == null ? 0 : table.Count);
+            table = table ?? new BlueTable(r.schema);
+            table.addRow(r);
+         }
+         ViewStream str = new BlueTableStream(table);
+         str = new SchemaGuesserStream(str);
+         mySetRealSource(str);
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
+
+
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
+         int n = 1;
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
+            if (columnAndType.Length == 1)
+               ci[n] = new ColumnInfo(c, n, typeof(double));
+            else
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
+            n++;
+         }
+
+
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
+         n = 0;
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
+                  continue;
+               t.addRow(
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
+               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
+            }
+
+         mySetRealSource(new BlueRowCollectionStaticStream(t));
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -0,0 +1,89 @@
+using System;
+
+using BCG.Utilities.General;
+using BCG;
+using bts;
+
+namespace bts.BlueOrb {
+   [DataStreamUserRespresentation("MillisecondStream")]
+   public class MillisecondStream : ViewStream, IWorker {
+      private DateTime _last_fire_time = DateTime.MinValue;
+      private readonly TimeSpan _period;
+      private readonly int _size;
+      private readonly ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema;
+      private bts.CircularBufferTable _table;
+
+      public MillisecondStream(int interval_msecs, int size)
+      : this(ViewStreamDatabase.instance, interval_msecs, size) {
+      }
+
+      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+         _vsd = vsd;
+         _period = TimeSpan.FromMilliseconds(interval_msecs);
+         _size = size;
+         _schema = BlueDataUtility.form_schema("Milliseconds",
+               Tuple.Create("id", TypeCode.Int32),
+               Tuple.Create("time", TypeCode.DateTime),
+               Tuple.Create("display_time", TypeCode.String));
+         _table = new CircularBufferTable(_schema, _size);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private int _id = 0;
+      public void doWork() {
+         if (DateTime.Now - _last_fire_time >= _period) {
+            _last_fire_time = DateTime.Now;
+            IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
+            _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
+            _table.add_row(nr);
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+   }
+   public class MSecondStream : TimeUnitStream {
+      public MSecondStream(int intervalInms, int size)
+         : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
+
+      protected override DateTime myRoundTime(DateTime dt) { return dt; }
+   }
+
+   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+      private IBlueTableSchema mySchema = new BlueTableSchema("ms",
+                                                               new ColumnInfo("id", 0, typeof(int)),
+                                                               new ColumnInfo("time", 1, typeof(string)));
+      public DisplayableMillisecondStream(int intervalInMs, int sz) {
+         mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
+      }
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         DateTime time = row.get_datetime("time");
+         var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
+         return new BlueRow(mySchema, row["id"], time_txt);
+      }
+
+      #endregion
+
+      private string myToString(DateTime dt) {
+         string sep = ":";
+         return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -0,0 +1,127 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   //using MathNet.Numerics.Statistics;
+
+
+   public enum MultiRowOp {
+      Mean,
+      Median,
+      Mode,
+      Range,
+      StdDev,
+      Max,
+      Min,
+      First,
+      Last,
+      Sum,
+      EdgeDiff,
+      EdgeDiffPercentage,
+      //IsEdgeDiff,
+   }
+   public class MultiRowOpHelper {
+      public readonly List<Tuple<string, MultiRowOp>> operations;
+
+      public static MultiRowOpHelper parse_ops(string text) {
+         // text is inp_col: op, inp_col: op, inp_col:op
+         List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
+         string[] calcs = text.Split(',');
+         foreach (string c in calcs) {
+            string[] toks = c.Split(':');
+            string inp_col = toks[0].Trim();
+            string op_txt = toks[1].Trim();
+            MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
+            rv.Add(Tuple.Create(inp_col, op));
+         }
+         return new MultiRowOpHelper(rv);
+      }
+
+      public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
+         operations = operations_;
+      }
+
+      // aggregates multiple rows into one - ie does group_by compute
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+
+         IBlueRow ir = rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
+      // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
+
+      public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
+         if (rolling_rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rolling_rows.ElementAt(0);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+         }
+         return or;
+      }
+
+      public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
+         switch (op) {
+            case MultiRowOp.Max:
+               return col_vals.Max<double>();
+            case MultiRowOp.Min:
+               return col_vals.Min<double>();
+            case MultiRowOp.Mean:
+               return col_vals.Sum() / col_vals.Count();
+            case MultiRowOp.Sum:
+               return col_vals.Sum();
+            case MultiRowOp.First:
+               return col_vals.First();
+            case MultiRowOp.Last:
+               return col_vals.Last();
+            case MultiRowOp.StdDev:
+               return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
+            default:
+               // TODO
+               Log.assert(false);
+               return Double.NaN;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -0,0 +1,207 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using ProtoBuf;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+   using System.Linq;
+
+   #region mappings
+   internal class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+   #endregion
+
+
+#region Row with mapipings build in
+   internal class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+   }
+   #endregion
+
+   [DataStreamUserRespresentation("ProjectStream")]
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         // by default every col will show up, so if we have the projection as (a:a1)
+         // it will convert a to a1, 
+         // examples: if a schema has cols a b c
+         // project(parent_str, "a:aa ") -> aa b c
+         // project(parent_str, "a:aa !c") -> aa b
+         // project(parent_str, "a:aa !") -> aa
+         // project(parent_str, "a:aa b")) -> aa b
+         bool hide_all = false;
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               if (p_tok.Length == 1) {
+                  // if it is a "blanket "!"=> hide all cols except the one that
+                  // are stated explicitly
+                  hide_all = true;
+               } else {
+                  string removed_col = p_tok.Substring(1);
+                  removed_cols.Add(removed_col);
+               }
+            } else if (p_tok.Contains(":")) {
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
+            } else {
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
+            }
+         }
+
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            } else {
+               if (hide_all)
+                  continue;
+               //if (ci.colName == "id" || removed_cols.Count > 0) {
+               // unless columns are  explicitely hidden, all columns show up
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               //}
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SchemaGuesserStream.cs

@@ -0,0 +1,30 @@
+using BCG;
+
+namespace bts.BlueOrb {
+   public class SchemaGuesserStream : ViewStream {
+
+      private IBlueRowCollection _rows = null;
+      private ViewStream _vs;
+      public SchemaGuesserStream(ViewStream vs) {
+         _vs = vs;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vs.onChange += _on_data;
+      }
+      protected override void myOnLastSubscriber() {
+         _vs.onChange -= _on_data;
+      }
+
+      private void _on_data(object obj) {
+         DataPacket dp = obj as DataPacket;
+         if (_rows == null) {
+            _rows = BlueDataUtility.get_typed_table(dp.newRows);
+         }
+         var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SingleRowMathStream.cs

@@ -0,0 +1,67 @@
+using Bts.ArithmeticExpressions;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BCG;
+
+namespace bts.BlueOrb {
+
+   [DataStreamUserRespresentation("SingleRowMathStream")]
+   public class MathStream : ViewStream, IStreamMapper {
+      private IArithmeticProgram myExp;
+      private BooleanExpressionFilter _bool;
+      List<string> myExtraColumns = new List<string>();
+      BlueTableSchema mySchema;
+
+
+      public MathStream(ViewStream vs, string mathExpression)
+         : this(vs, mathExpression, string.Empty) {
+      }
+
+      public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
+         myExp = ArithmeticProgramParser.Parse(mathExpression) ??
+            throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
+         myExtraColumns = myExp.OutputIdentifiers.ToList();
+         if (!string.IsNullOrEmpty(boolean_expression))
+            _bool = new BooleanExpressionFilter(boolean_expression);
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
+            Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
+            foreach (ColumnInfo ci in inputSchema.columnInfo)
+               colIdx[ci.colName] = ci;
+
+            foreach (string ncn in myExtraColumns) {
+               ColumnInfo newC;
+               if (colIdx.ContainsKey(ncn))
+                  newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
+               else
+                  newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
+               colIdx[ncn] = newC;
+            }
+            mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row) {
+         MutableBlueRow result = new MutableBlueRow(mySchema, row);
+         if (_bool == null || _bool.filter(row)) {
+            myExp.Evaluate(result.AsDictionary());
+         }
+         return result;
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/TopStream.cs

@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("TopStream")]
+   public class TopStream : ViewStream { 
+      public TopStream(ViewStream vs, int n_rows, string sort_col) {
+         ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
+         mySetRealSource(rs);
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -0,0 +1,261 @@
+using System;
+using System.Collections.Generic;
+
+using mbts;
+using BCG;
+
+namespace bts.BlueOrb {
+   internal class MutableAppendTable : IBlueRowCollection {
+      private class MyRow : IBlueRow {
+         private IBlueRow myRow;
+         private int myID;
+
+         public MyRow(IBlueRow r, int id) {
+            myRow = r;
+            myID = id;
+         }
+
+         #region IBlueRow Members
+
+         public IBlueTableSchema schema { get { return myRow.schema; } }
+
+         public object this[int index] {
+            get {
+               if (myRow.schema[index].colName == "id")
+                  return myID;
+               else
+                  return myRow[index];
+            }
+         }
+
+         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+         #endregion
+
+         #region IDictionary Members
+
+         public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+         public void Clear() { throw new Exception("Row is read only."); }
+
+         public bool Contains(object key) { return myRow.Contains(key); }
+
+         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+         public bool IsFixedSize { get { return true; } }
+
+         public bool IsReadOnly { get { return true; } }
+
+         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+         public System.Collections.ICollection Values {
+            get {
+               object[] vals = new object[myRow.Count];
+               CopyTo(vals, 0);
+               return vals;
+            }
+         }
+
+         public object this[object key] {
+            get { return key is string ? this[key as string] : this[(int)key]; }
+            set { throw new Exception("The method or operation is not implemented."); }
+         }
+
+         #endregion
+
+         #region ICollection Members
+
+         public void CopyTo(Array array, int index) {
+            myRow.CopyTo(array, index);
+            array.SetValue(myID, index + myRow.schema["id"].colNo);
+         }
+
+         public int Count { get { return myRow.Count; } }
+
+         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
+
+         public object SyncRoot { get { return myRow.SyncRoot; } }
+
+         #endregion
+
+         #region IEnumerable Members
+
+         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+         #endregion
+      }
+
+      private BlueTable myRows;
+      private int myID = 0;
+      /// <summary>
+      /// maps the table no to a dictionary containing the
+      /// local IDs to a row in the final big table
+      /// </summary>
+      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+      public IBlueRow addRow(IBlueRow r, int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+         myRows.addRow(nr);
+         myIDMap[tableNo].Add((int)r["id"], nr);
+         return nr;
+      }
+
+      public IBlueRow removeRow(IBlueRow r, int tableNo) {
+         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+         myRows.deleteRow(dr);
+         myIDMap[tableNo].Remove((int)r["id"]);
+         return dr;
+      }
+
+      public IBlueRowCollection removeTable(int tableNo) {
+         if (!myIDMap.ContainsKey(tableNo))
+            return null;
+         else {
+            BlueTable rv = new BlueTable(myRows.schema);
+            foreach (IBlueRow r in myIDMap[tableNo].Values)
+               rv.addRow(r);
+            myRows.deleteRows(rv);
+            myIDMap.Remove(tableNo);
+            return rv;
+         }
+      }
+
+      public void clear() {
+         myRows.clear();
+         myIDMap.Clear();
+      }
+
+      #region IBlueRowCollection Members
+
+      public IBlueRow this[int id] { get { return myRows[id]; } }
+
+      public IBlueTableSchema schema { get { return myRows.schema; } }
+
+      public bool containsID(int id) { return myRows.containsID(id); }
+
+      #endregion
+
+      #region ICollection Members
+
+      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+      public int Count { get { return myRows.Count; } }
+
+      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+      public object SyncRoot { get { return myRows.SyncRoot; } }
+
+      #endregion
+
+      #region IEnumerable Members
+
+      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+      #endregion
+   }
+   /// <summary>
+   /// MutableStream can append multiple streams
+   /// as well as give the user the flexibility
+   /// to append (or remove) more streams at
+   /// runtime
+   /// </summary>
+   }
+namespace bts.BlueOrb { 
+   [DataStreamUserRespresentation("UnionStream")]
+   public class MutableAppendStream : ViewStream {
+      private bool mySubscriptionActive = false;
+      private MutableAppendTable myTable;
+      private int myMovingIndex;
+      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
+      private bool _allign_col_orders = true;
+
+      public MutableAppendStream(params ViewStream[] streams)
+         : this(new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
+         : this(allign_col_order, new List<ViewStream>(streams)) { }
+
+      public MutableAppendStream(ICollection<ViewStream> streams)
+         : this(true, streams) { }
+
+      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
+         _allign_col_orders = allign_col_orders;
+         foreach (ViewStream vs in streams) {
+            myMovingIndex++;
+            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+            if (_allign_col_orders)
+               pj = new ProjectStream(pj, "!");
+            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         }
+      }
+
+      public void addStream(ViewStream vs) {
+         myMovingIndex++;
+         ///the change name is done to identify the callbacks, and the project to
+         ///make all the schemas col orders the same
+         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
+         if (_allign_col_orders)
+            pj = new ProjectStream(pj, "!");
+         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
+         if (mySubscriptionActive)
+            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      public void removeStream(ViewStream vs) {
+         int index = -1;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
+            if (p.Value.first == vs) {
+               index = p.Key;
+               if (mySubscriptionActive) {
+                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+                  if (myTable != null) {
+                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
+                     if (delRows != null)
+                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
+                  }
+               }
+               break;
+            }
+            if (index > 0)
+               myStreamIndex.Remove(index);
+         }
+      }
+
+      protected override void myOnFirstSubscriber() {
+         mySubscriptionActive = true;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
+      }
+
+      protected override void myOnLastSubscriber() {
+         mySubscriptionActive = false;
+         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
+            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
+         if (myTable != null)
+            myTable.clear();
+      }
+
+      private void myHandleStreamChanges(object newValue) {
+         DataPacket dp = newValue as DataPacket;
+         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
+         BlueTable adds = new BlueTable(myTable.schema),
+                     dels = new BlueTable(myTable.schema);
+         int index = Int32.Parse(dp.schema.tableName);
+         foreach (IBlueRow r in dp.deletedRows) {
+            IBlueRow dr = myTable.removeRow(r, index);
+            dels.addRow(dr);
+         }
+         foreach (IBlueRow r in dp.addedRows) {
+            IBlueRow nr = myTable.addRow(r, index);
+            adds.addRow(nr);
+         }
+         myFireEvent(new DataPacket(myTable, adds, dels));
+      }
+   }
+}

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -18,6 +18,7 @@
 
   <ItemGroup>
     <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -1,52 +1,47 @@
 using System;
 
-namespace BCG
-{
-   [DataStreamUserRespresentation ("count")]
-	public class CountStream : ViewStream
-	{
+namespace bts.BlueOrb {
+   using BCG;
+   [DataStreamUserRespresentation("CountStream")]
+   public class CountStream : ViewStream {
       private DataPacket myLastFiredDataPacket;
-      private IBlueTableSchema mySchema = new BlueTableSchema ("count", 
-         new ColumnInfo ("id", 0, typeof(int)),
-         new ColumnInfo ("count", 1, typeof(int)));
+      private IBlueTableSchema mySchema = new BlueTableSchema("count",
+         new ColumnInfo("id", 0, typeof(int)),
+         new ColumnInfo("count", 1, typeof(int)));
       private int myId = 0;
       private ViewStream myVS;
 
-      public CountStream (ViewStream vs)
-      {
+      public CountStream(ViewStream vs) {
          myVS = vs;
       }
 
-      public CountStream (string tableName)
-         : this (new TableStream (tableName))
-      {}
+      public CountStream(string tableName)
+         : this(new TableStream(tableName)) { }
 
-      protected override void myOnFirstSubscriber () {
+      protected override void myOnFirstSubscriber() {
          myVS.onChange += new ChangeDelegate(myHandleChanges);
       }
-      protected override void myOnLastSubscriber () {
+      protected override void myOnLastSubscriber() {
          myVS.onChange -= new ChangeDelegate(myHandleChanges);
       }
 
 
-      private void myHandleChanges(object newValue)
-      {
-         myFireEvent (
+      private void myHandleChanges(object newValue) {
+         myFireEvent(
             myLastFiredDataPacket =
                myLastFiredDataPacket == null ?
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
-                  new BlueTable (mySchema))
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
+                  new BlueTable(mySchema))
                   :
-                  new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId), 
+                  new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
                   myLastFiredDataPacket.newRows)
             );
       }
 
-      private IBlueRowCollection myGetRows (object dp, int id)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
+      private IBlueRowCollection myGetRows(object dp, int id) {
+         BlueTable retVal = new BlueTable(mySchema);
          retVal.addRow
-            (new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
+            (new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -5,25 +5,20 @@ using mbts.config;
 using System;
 using System.Text.RegularExpressions;
 
-namespace BCG.StreamWorld.Addition
-{
-   [DataStreamUserRespresentation (false)]
-   public class FilterStream : ViewStream
-   {
+namespace BCG.StreamWorld.Addition {
+   [DataStreamUserRespresentation(false)]
+   public class FilterStream : ViewStream {
       private ViewStream myBaseStream;
       private BlueTable myBaseTable, myStateTable;
       private IStreamFilter myFilter;
 
       public FilterStream(ViewStream vs)
-         : this(vs, "1=1")
-      { }
+         : this(vs, "1=1") { }
 
       public FilterStream(ViewStream vs, string filterExp)
-         : this(vs, new BooleanExpressionFilter(filterExp))
-      { }
+         : this(vs, new BooleanExpressionFilter(filterExp)) { }
 
-      public FilterStream(ViewStream vs, IStreamFilter filter)
-      {
+      public FilterStream(ViewStream vs, IStreamFilter filter) {
          myBaseStream = vs;
          myFilter = filter;
          if (filter is IDynamicStreamFilter)
@@ -31,43 +26,35 @@ namespace BCG.StreamWorld.Addition
       }
 
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
          myBaseTable.clear();
          myStateTable.clear();
       }
 
-      private void myHandleChanges(object newValue)
-      {
+      private void myHandleChanges(object newValue) {
          DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null)
-         {
+         if (myStateTable == null) {
             bts.Log.assert(myBaseTable == null);
             myStateTable = new BlueTable(dp.schema);
             myBaseTable = new BlueTable(dp.schema);
          }
          BlueTable dels = new BlueTable(dp.schema),
                      adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"]))
-            {
+            if (myStateTable.containsID((int)r["id"])) {
                dels.addRow(r);
                myStateTable.deleteRow(r);
             }
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             myBaseTable.addRow(r);
-            if (myFilter.filter(r))
-            {
+            if (myFilter.filter(r)) {
                adds.addRow(r);
                myStateTable.addRow(r);
             }
@@ -76,13 +63,10 @@ namespace BCG.StreamWorld.Addition
             myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
-      public IStreamFilter filter
-      {
-         set
-         {
+      public IStreamFilter filter {
+         set {
             myFilter = value;
-            if (myBaseTable != null)
-            {
+            if (myBaseTable != null) {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
                foreach (IBlueRow r in myBaseTable)

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Linq;
 
 namespace BCG {
-   public class GroupByStream : MethodDrivenStream {
+   internal class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs

@@ -74,7 +74,7 @@ namespace BCG {
       }
 
       public void _on_stream_cb(object dp_obj) {
-         Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
+         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
          var timer = Stopwatch.StartNew();
          DataPacket dp = (DataPacket)dp_obj;
          _fill_missing_data(dp);

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -50,7 +50,7 @@ namespace BCG {
          int data_col_no = 1;
          for (int jp = 0; jp < _join_specs.Length; jp++) {
             NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
+            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
             data_col_no += _join_specs[jp].data_col_names.Length;
             s.on_new_data += _stream_callback;
             _book_keeping_data.Add(s);
@@ -71,7 +71,7 @@ namespace BCG {
       }
 
       private void _stream_callback() {
-         Log.Gen.info("NonIncJoinStream._stream_callback received callback");
+         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
          _is_dirty = true;
          doWork();
       }
@@ -99,7 +99,7 @@ namespace BCG {
 
       private int _n_total_fires = 0, _n_expensive_fires = 0;
       private void _form_and_fire() {
-         Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
          if (_is_ready()) {
             _construct_schema();
             var timer = Stopwatch.StartNew();
@@ -126,11 +126,11 @@ namespace BCG {
             _n_total_fires++;
             //if (timer.ElapsedMilliseconds >= 250) {
             _n_expensive_fires++;
-            string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-               + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            Log.Gen.info(log_msg);
+            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+            //Log.Gen.info(log_msg);
             //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
+            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
             myFireEvent(dp_out);
             //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
          }
@@ -152,7 +152,7 @@ namespace BCG {
       }
 
       public void doWork() {
-         Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
+         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
          if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
             _is_dirty = false;
             _last_fire_time = DateTime.Now;

src/DataStreamSuite/DataStream/Primitives/Time/SecondStream.cs

@@ -37,4 +37,42 @@ namespace BCG
       }
          
    }
+
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd = vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj as DataPacket;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRows, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+
 }

src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream

@@ -0,0 +1,40 @@
+namespace bts.SilverPine{
+   using BGC;
+   using System;
+   [DataStreamUserRespresentation("throttle")]
+   public class ThrottleStream : ViewStream{
+      private readonly ViewStream _main_stream;
+      private readonly TimeSpan _ts;
+      private readonly ViewStreamDatabase _vsd;
+      private DateTime _last_fire_time = DateTime.MinValue;
+
+      public ThrottleStream (ViewStream main_stream, int throttle_seconds)
+      :
+      this(main_stream, TimeSpan.FromSeconds(throttle_seconds), ViewStreamDatabase.instance)
+      {
+      }
+
+      public ThrottleStream (ViewStream main_stream, TimeSpan ts, ViewStreamDatabase vsd){
+         _main_stream = main_stream;
+         _ts = ts;
+         _vsd;
+      }
+
+       protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _handle_base_stream_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _handle_base_stream_update;
+      }
+
+      private void _handle_base_stream_update(object dp_obj){
+         if (DateTime.Now - _last_fire_time >= _ts){
+            DataPacket dp_in = dp_obj;
+            _last_fire_time = DateTime.Now;
+            DataPacket dp = BlueDataUtility.build_dp_inc(dp_in.newRow, myLastValue as DataPacket);
+            myFireEvent(dp);
+         }
+      }
+ }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -40,4 +40,4 @@ namespace BCG {
             myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
         }
     }
-}
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -280,6 +280,28 @@ namespace BCG {
 
       #region json
 
+      public static void write_to_json_file(this IBlueRowCollection table, string file_path) { 
+         write_dict_to_json_file(file_path, table.to_namevalue_pairs());
+      }
+
+      public static void write_dict_to_json_file(string file_path, List<Dictionary<string, string>> data) {
+         var options = new JsonSerializerOptions { WriteIndented = true };
+         string json = System.Text.Json.JsonSerializer.Serialize(data, options);
+         File.WriteAllText(file_path, json);
+      }
+
+      public static List<Dictionary<string, string>> ReadFromFile(string filePath) {
+         string json = File.ReadAllText(filePath);
+         return System.Text.Json.JsonSerializer.Deserialize<List<Dictionary<string, string>>>(json);
+      }
+
+      public static void write_table_to_json(IBlueRowCollection table) {
+         List<Dictionary<string, string>> rows = new List<Dictionary<string, string>>();
+         foreach (IBlueRow row in table) {
+            row.to_json();
+         }
+      }
+
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
          Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
          return dictionary;
@@ -348,6 +370,14 @@ namespace BCG {
          }
          return rv;
       }
+
+      public static List<Dictionary<string,string>> to_namevalue_pairs(this IBlueRowCollection table) {
+         List<Dictionary<string, string>> rv = new List<Dictionary<string, string>>();
+         foreach (IBlueRow r in table) {
+            rv.Add(r.to_dict());
+         }
+         return rv;
+      }
       #region Schema types
       // these are the only types allowed in schemas
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
@@ -1225,6 +1255,20 @@ namespace BCG {
          return rv;
       }
 
+      public static Dictionary<string, string> to_dict(this IBlueRow r) {
+         Dictionary<string, string> map = new Dictionary<string, string>();
+         foreach (ColumnInfo c in r.schema.columnInfo) {
+            map.Add(c.colName, r[c.colName].ToString());
+         }
+         return map;
+      }
+
+      public static string to_json(this IBlueRow r) {
+         Dictionary<string, string> map = r.to_dict();
+         string json = System.Text.Json.JsonSerializer.Serialize(map);
+         return json;
+      }
+
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
          if (old_rows == null || old_rows.Count == 0) {

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -47,10 +47,10 @@ namespace BCG {
       /// </summary>
       /// <param name="assemblyName"></param>
       public void loadAssembly(string assemblyName) {
-         Func<Type, bool> is_silverpine_only = (t) => {
-            if (!assemblyName.Contains("SharedStreams.dll"))
-               return true;
-            return t.Namespace == "bts.SilverPine";
+         Func<Type, bool> is_blue_orb = (t) => {
+            //if (!assemblyName.Contains("SharedStreams.dll"))
+            //   return true;
+            return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
          };
          bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
@@ -61,7 +61,7 @@ namespace BCG {
             Assembly.LoadFrom(assemblyName);
 
          foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface && is_silverpine_only(type)
+            if (!type.IsInterface && is_blue_orb(type)
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
                && type.IsPublic) {
@@ -77,7 +77,7 @@ namespace BCG {
                      break;
                   }
                if (isVisible) {
-                  Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
                   myTypeLookupTable[userRep] = type;
                }
             }

src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs

@@ -1,4 +1,5 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
+   using bts.SilverPine;
    using BCG;
 
    using System.Runtime.CompilerServices;
@@ -45,7 +46,7 @@ namespace bts.SilverPine {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1DayBars")]
-   public class FuturesHistorical1DayBarsStream : ViewStream {
+   internal class FuturesHistorical1DayBarsStream : ViewStream {
 
       // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?)
       public FuturesHistorical1DayBarsStream(string symbol) :
@@ -103,7 +104,7 @@ namespace bts.SilverPine {
    }
 
    [DataStreamUserRespresentation("FuturesHistorical1MinBars")]
-   public class FuturesHistorical1MinuteBarsStream : ViewStream {
+   internal class FuturesHistorical1MinuteBarsStream : ViewStream {
       // same as the above but for one min
       // this gets the raw bars for a symbol
       public FuturesHistorical1MinuteBarsStream(string symbol) :
@@ -187,46 +188,6 @@ namespace bts.SilverPine {
       }
    }
 
-   [DataStreamUserRespresentation("FuturePriceDiff")]
-   public class FutureHistoricalBarsPriceDiffStream : ViewStream {
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) {
-      }
-
-      public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col)
-      :
-      this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) {
-      }
-
-
-      public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) {
-         BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN;
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2));
-
-         rs = new ColumnDifferenceStream(rs, r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt1.Equals(r_dt);
-            return rv;
-         },
-         //r => dt2.Equals(r.get_datetime("ts_event")), col, col);
-         r => {
-            DateTime r_dt = r.get_datetime("ts_event");
-            var rv = dt2.Equals(r_dt);
-            return rv;
-         }, col, col);
-         rs = new NewProjectStream(rs, "diff !");
-         mySetRealSource(rs);
-      }
-   }
-
    // takes the contract name and expands it into multuiple columns corresponding to the contracts.
    // the contract columns are a union, of all properties with some of them empty
    public class ContractExpanderStream : ViewStream {
@@ -247,3 +208,29 @@ namespace bts.SilverPine {
    }
 
 }
+
+namespace bts.BlueOrb {
+   using BCG;
+
+   using bts.api;
+
+   using Newtonsoft.Json.Serialization;
+
+   [DataStreamUserRespresentation("FuturesContinousContract")]
+   public class FuturesContinousContractStream : ViewStream {
+      // the argument is symbol, like "ES" etc and not the contract itself
+      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
+      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
+         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
+         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
+         r = new ContractExpanderStream(r, "symbol");
+         r = new FilterStream(r, "contract_type='Future'");
+         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
+         // with the open interest stream on symbol/contract
+         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
+         r = new NewProjectStream(r, "ts_event symbol !");
+         r = new SortStream(r, "-ts_event");
+         mySetRealSource(r);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs

@@ -0,0 +1,18 @@
+namespace bts.BlueOrb {
+   using bts.SilverPine;
+   using BCG;
+   using System;
+
+   [DataStreamUserRespresentation("SpatialArbScoreStream")]
+   public class SpatialArbScoreStream : ViewStream {
+      public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
+         : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
+
+      public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
+         ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
+         rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
+         rs = new SortStream(rs, "-n_trade_ticks");
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs

@@ -0,0 +1,113 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using SharpCompress.Common.Rar;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   // For all the pairs generated by SpatialFuturePairsStream, this stream 
+   // generates the statistics, by appending together a SpreadStatsStream
+   [DataStreamUserRespresentation("SpatialArbStatsStream")]
+   public class SpatialArbStatsStream : ViewStream{
+      private ViewStream _spatial_pairs_stream;
+      private ViewStreamDatabase _vsd;
+      private ViewStream _tot_str = null;
+      private HashSet<string> _key_map = new HashSet<string>();
+      private ViewStream _bbo_stream;
+      private readonly TimeSpan _window_sz, _sampling_period_sz;
+
+      public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
+         : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
+
+      public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
+         _vsd = vsd;
+         _window_sz = window_sz;
+         _sampling_period_sz = sampling_period_sz;
+         _spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
+         _bbo_stream = new BBOStream(vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spatial_pairs_stream.onChange += _spatial_pairs_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spatial_pairs_stream.onChange -= _spatial_pairs_stream_onchange;
+      }
+
+      private ViewStream _build_mapped_stream(ViewStream parent_str) {
+         Func<IBlueTableSchema, IBlueTableSchema> build_schema = (inp_schema) => {
+            IBlueTableSchema rv = new BlueTableSchema("SpreaddStats", new ColumnInfo("id", 0, TypeCode.Int32));
+            rv = rv.add_column(TypeCode.String, "spread");
+            rv = rv.add_column(TypeCode.String, "window");
+            // bids
+            rv = rv.add_column(TypeCode.Double, "bid_mean");
+            rv = rv.add_column(TypeCode.Double, "bid_stddev");
+            rv = rv.add_column(TypeCode.Double, "bid_skewness");
+            rv = rv.add_column(TypeCode.Double, "bid_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "bid_max");
+            rv = rv.add_column(TypeCode.Double, "bid_min");
+            // asks
+            rv = rv.add_column(TypeCode.Double, "ask_mean");
+            rv = rv.add_column(TypeCode.Double, "ask_stddev");
+            rv = rv.add_column(TypeCode.Double, "ask_skewness");
+            rv = rv.add_column(TypeCode.Double, "ask_kurtosis");
+            rv = rv.add_column(TypeCode.Double, "ask_max");
+            rv = rv.add_column(TypeCode.Double, "ask_min");
+            // tail distribution
+            rv = rv.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+            rv = rv.add_column(TypeCode.Double, "ask Z < -2σ");
+            // trading ticks
+            rv = rv.add_column(TypeCode.Int32, "n_buy_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_buy_price");
+            rv = rv.add_column(TypeCode.Int32, "n_sell_ticks");
+            rv = rv.add_column(TypeCode.Double, "cumm_sell_price");
+            rv = rv.add_column(TypeCode.Int32, "n_trade_ticks");
+            return rv;
+         };
+         Func<IBlueRow, IBlueRow> build_row = (r) => {
+            IBlueTableSchema s = build_schema(r.schema);
+            return new BlueRow(s, r["id"], r["spread"], r["window"],
+                           r["bid_mean"], r["bid_stddev"], r["bid_skewness"], r["bid_kurtosis"], r["bid_max"], r["bid_min"],
+                           r["ask_mean"], r["ask_stddev"], r["ask_skewness"], r["ask_kurtosis"], r["ask_max"], r["ask_min"],
+                           r["bid Z ≥ 2σ"], r["ask Z < -2σ"],
+                           r["n_buy_ticks"], r["cumm_buy_price"], r["n_sell_ticks"], r["cumm_sell_price"], r["n_trade_ticks"]);
+         };
+         MapStream rs = new MapStream(parent_str, inp_schema => build_schema(inp_schema),
+            inp_r => build_row(inp_r));
+         return rs;
+      }
+
+      private void _spatial_pairs_stream_onchange(object dp_obj) {
+         if (_tot_str != null)
+            return;
+         MutableAppendStream append_str = new MutableAppendStream();
+         _tot_str = _build_mapped_stream(append_str);
+         //_tot_str = new ThrottleStream(_tot_str, 1000);
+         //_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
+         _tot_str.onChange += _handle_tot_stream;
+
+         DataPacket dp = dp_obj as DataPacket;         
+         foreach (IBlueRow r in dp.newRows) {
+            string s1 = r.get<string>("s1");
+            string s2 = r.get<string>("s2");
+            string key = SpatialFuturePairsStream.get_key(s1, s2);
+            if (!_key_map.Contains(key)) {
+               _key_map.Add(key);
+               ViewStream spread_stats_stream = new SpreadStatsStream(s1, s2, 1, 1, Convert.ToInt32(_window_sz.TotalSeconds), 
+                                                                     Convert.ToInt32(_sampling_period_sz.TotalSeconds), _bbo_stream, _vsd);
+               append_str.addStream(spread_stats_stream);
+            }
+         }
+      }
+
+      private void _handle_tot_stream(object dp_obj) {
+         myFireEvent(dp_obj);
+      }
+   }
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs

@@ -0,0 +1,89 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using bts.api;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SpatialFuturePairsStream")]
+   public class SpatialFuturePairsStream : ViewStream {
+      private ViewStream _main_stream;
+      private IBlueTableSchema _schema;
+      private HashSet<string> _key_map = new HashSet<string>();
+      public SpatialFuturePairsStream()
+         : this(ViewStreamDatabase.instance) { }
+
+      public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
+         _main_stream = new InstrumentsStream(vsd);
+         _main_stream = new FilterStream(_main_stream, "sec_type='Future'");
+         _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "s1");
+         _schema = _schema.add_column(TypeCode.String, "venue_1");
+         _schema = _schema.add_column(TypeCode.String, "s2");
+         _schema = _schema.add_column(TypeCode.String, "venue_2");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _main_stream.onChange += _main_stream_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= _main_stream_onchange;
+      }
+
+      public static string get_key(string s1, string s2) {
+         string key = string.Compare(s1, s2, StringComparison.Ordinal) <= 0 ? s1 + s2 : s2 + s1;
+         return key;
+      }
+
+      public static string get_key(IBlueRow r1, IBlueRow r2) {
+         string s1 = r1.get<string>("sec");
+         string s2 = r2.get<string>("sec");
+         return get_key(s1, s2);
+      }
+
+      private bool _combine(IBlueRow r1, IBlueRow r2) {
+         string key = get_key(r1, r2);
+         if (_key_map.Contains(key))
+            return false;
+         if (r1.get<string>("symbol") != r2.get<string>("symbol"))
+            return false;
+         if (r1.get_enum<Venue>("venue") == r2.get_enum<Venue>("venue"))
+            return false;
+         if (r1.get<int>("sec_key") == r2.get<int>("sec_key")) {
+            return false;
+         }
+         DateTime e1 = r1.get_datetime("expiration");
+         DateTime e2 = r2.get_datetime("expiration");
+         bool exps_eq = e1 == e2 || (e1.Year >= 2100 && e2.Year >= 2100);
+         if (!exps_eq)
+            return false;         
+         return true;
+      }
+
+      private void _main_stream_onchange(object dp_obj) {
+         _key_map.Clear();
+         DataPacket dp = dp_obj as DataPacket;
+         BlueTable table = new BlueTable(_schema);
+         var rows = dp.newRows.to_list();
+         foreach (var r1 in rows) {
+            foreach (var r2 in rows) {
+               if (_combine(r1, r2)) {
+                  IBlueRow nr = new BlueRow(_schema, table.Count, r1.get<string>("sec"), r1.get<string>("venue"),
+                     r2.get<string>("sec"), r2.get<string>("venue"));
+                  table.addRow(nr);
+                  string key = get_key(r1, r2);
+                  _key_map.Add(key);
+               }
+            }
+         }
+         var dp_out = BlueDataUtility.build_dp(table, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs

@@ -0,0 +1,656 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearRegression;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using MathNet.Numerics.LinearAlgebra;
+using Amazon.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+// stats calculator
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+
+/// <summary>
+/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
+/// Assumes the input spread data is a trailing, time-causal window of recent ticks (e.g., last 120 seconds).
+/// </summary>
+public static class SpreadPerformanceEvaluator
+{
+
+   public static class SpreadStatistics {
+      public class SpreadStatsResult {
+         public Dictionary<string, double> bid_zscore_distribution { get; set; }
+         public Dictionary<string, double> ask_zscore_distribution { get; set; }
+         public Dictionary<string, double> mid_zscore_distribution { get; set; }
+
+         public DescriptiveStatistics bid_stats { get; set; }
+         public DescriptiveStatistics ask_stats { get; set; }
+         public DescriptiveStatistics mid_stats { get; set; }
+      }
+
+      private static readonly string[] _zscore_labels = new[]
+      {
+        "Z < -2σ",
+        "-2σ ≤ Z < -1σ",
+        "-1σ ≤ Z < 0σ",
+        "0σ ≤ Z < 1σ",
+        "1σ ≤ Z < 2σ",
+        "Z ≥ 2σ"
+    };
+
+      public static SpreadStatsResult analyze_spread_quotes(List<(DateTime timestamp, double bid, double bid_qty, double ask, double ask_qty)> quotes) {
+         if (quotes == null || quotes.Count == 0)
+            throw new ArgumentException("Spread quote list cannot be null or empty.");
+
+         var bid_prices = quotes.Select(q => q.bid).ToList();
+         var ask_prices = quotes.Select(q => q.ask).ToList();
+         var mid_prices = quotes.Select(q => (q.bid + q.ask) / 2.0).ToList();
+
+         var bid_stats = new DescriptiveStatistics(bid_prices);
+         var ask_stats = new DescriptiveStatistics(ask_prices);
+         var mid_stats = new DescriptiveStatistics(mid_prices);
+
+         var bid_zscores = bid_prices.Select(x => zscore(x, bid_stats.Mean, bid_stats.StandardDeviation)).ToList();
+         var ask_zscores = ask_prices.Select(x => zscore(x, ask_stats.Mean, ask_stats.StandardDeviation)).ToList();
+         var mid_zscores = mid_prices.Select(x => zscore(x, mid_stats.Mean, mid_stats.StandardDeviation)).ToList();
+
+         var bid_distribution = bin_zscores_to_percentages(bid_zscores);
+         var ask_distribution = bin_zscores_to_percentages(ask_zscores);
+         var mid_distribution = bin_zscores_to_percentages(mid_zscores);
+
+         return new SpreadStatsResult {
+            bid_zscore_distribution = bid_distribution,
+            ask_zscore_distribution = ask_distribution,
+            mid_zscore_distribution = mid_distribution,
+            bid_stats = bid_stats,
+            ask_stats = ask_stats,
+            mid_stats = mid_stats
+         };
+      }
+
+      private static double zscore(double x, double mean, double std_dev) {
+         return std_dev == 0 ? 0 : (x - mean) / std_dev;
+      }
+
+      private static Dictionary<string, double> bin_zscores_to_percentages(List<double> zscores) {
+         var bins = _zscore_labels.ToDictionary(label => label, _ => 0);
+
+         foreach (var z in zscores) {
+            if (z < -2) bins["Z < -2σ"]++;
+            else if (z < -1) bins["-2σ ≤ Z < -1σ"]++;
+            else if (z < 0) bins["-1σ ≤ Z < 0σ"]++;
+            else if (z < 1) bins["0σ ≤ Z < 1σ"]++;
+            else if (z < 2) bins["1σ ≤ Z < 2σ"]++;
+            else bins["Z ≥ 2σ"]++;
+         }
+
+         double total = zscores.Count;
+         return bins.ToDictionary(kvp => kvp.Key, kvp => total == 0 ? 0 : (kvp.Value / total) * 100.0);
+      }
+
+      public static double score_spread(SpreadStatsResult result) {
+         var ask_bins = result.ask_zscore_distribution;
+         var bid_bins = result.bid_zscore_distribution;
+         var mid_bins = result.mid_zscore_distribution;
+
+         double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"];
+         double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"];
+         double opportunity_score = buy_opportunity + sell_opportunity;
+
+         double below_mean = mid_bins["Z < -2σ"] + mid_bins["-2σ ≤ Z < -1σ"] + mid_bins["-1σ ≤ Z < 0σ"];
+         double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"];
+         double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6);
+
+         double amplitude_score = result.mid_stats.StandardDeviation;
+         double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness));
+         double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0;
+
+         double score =
+             opportunity_score *
+             symmetry_score *
+             amplitude_score *
+             skew_penalty *
+             (1.0 + kurtosis_bonus * 0.1);
+
+         return score;
+      }
+   }
+
+
+
+
+
+/// <summary>
+/// Represents a single trade action (entry or exit) taken during the simulation.
+/// </summary>
+public struct TradeAction
+    {
+        public bool IsEntry { get; set; }    // True if this action opened a position
+        public bool IsLong { get; set; }     // True = Long, False = Short
+        public double Price { get; set; }    // Executed price
+        public int Index { get; set; }       // Tick index of the action
+
+        public override string ToString()
+        {
+            string side = IsLong ? "Long" : "Short";
+            string type = IsEntry ? "Open" : "Close";
+            return $"{type} {side} @ {Price:F4} (Tick {Index})";
+        }
+    }
+
+    /// <summary>
+/// Simulates a mean-reversion strategy using rolling, causal mid-price stats.
+/// ✅ Entry:
+///   - Buy at ask when ask < rollingMean - entryThreshold × rollingStdDev
+///   - Sell at bid when bid > rollingMean + entryThreshold × rollingStdDev
+/// ✅ Exit:
+///   - Close when mid returns within rollingMean ± exitThreshold × rollingStdDev
+///   - Long exits at bid, short exits at ask
+/// This version uses only past and current data at each step (no forward-looking bias).
+/// </summary>
+public static (double TotalPnL, List<TradeAction> TradeActions, int BuyTurns, int SellTurns) EvaluateSimulatedPnL(
+    List<Tuple<double, double, double, double>> spreadData,
+    int statWindow = 30,
+    double entryThresholdMultiplier = 0.5,
+    double exitThresholdMultiplier = 0.1)
+{
+    var trades = new List<TradeAction>();
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    if (spreadData.Count < statWindow)
+        return (0.0, trades, 0, 0);
+
+    bool positionOpen = false;
+    bool isLong = false;
+    double entryPrice = 0.0;
+    double pnl = 0.0;
+    int buyTurns = 0;
+    int sellTurns = 0;
+
+    for (int i = statWindow; i < spreadData.Count; i++)
+    {
+        double bid = spreadData[i].Item1;
+        double ask = spreadData[i].Item3;
+        double mid = mids[i];
+
+        // Rolling window
+        var window = mids.Skip(i - statWindow).Take(statWindow).ToList();
+        double mean = window.Average();
+        double stddev = Math.Sqrt(window.Select(x => Math.Pow(x - mean, 2)).Average());
+
+        double lowerThreshold = mean - entryThresholdMultiplier * stddev;
+        double upperThreshold = mean + entryThresholdMultiplier * stddev;
+        double exitBand = exitThresholdMultiplier * stddev;
+
+        if (!positionOpen)
+        {
+            if (ask < lowerThreshold)
+            {
+                positionOpen = true;
+                isLong = true;
+                entryPrice = ask;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = true, Price = entryPrice, Index = i });
+            }
+            else if (bid > upperThreshold)
+            {
+                positionOpen = true;
+                isLong = false;
+                entryPrice = bid;
+                trades.Add(new TradeAction { IsEntry = true, IsLong = false, Price = entryPrice, Index = i });
+            }
+        }
+        else
+        {
+            if (Math.Abs(mid - mean) < exitBand)
+            {
+                double exitPrice = isLong ? bid : ask;
+                pnl += isLong ? (exitPrice - entryPrice) : (entryPrice - exitPrice);
+
+                trades.Add(new TradeAction { IsEntry = false, IsLong = isLong, Price = exitPrice, Index = i });
+
+                if (isLong) buyTurns++;
+                else sellTurns++;
+
+                positionOpen = false;
+            }
+        }
+    }
+
+    return (pnl, trades, buyTurns, sellTurns);
+}
+
+
+
+    /// <summary>
+    /// Estimates the Hurst exponent of the spread mid-prices.
+    /// Assumes the input is a trailing window of recent data.
+    /// Hurst < 0.5 implies mean-reversion; ideal is 0.35–0.45.
+    /// </summary>
+    public static double EvaluateHurstExponent(List<Tuple<double, double, double, double>> spreadData)
+    {
+        const int MinLength = 21;
+
+        if (spreadData == null || spreadData.Count < MinLength)
+            return double.NaN;
+
+        var spreadMids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+        int n = spreadMids.Length;
+
+        var lags = new List<double>();
+        var tau = new List<double>();
+
+        for (int lag = 2; lag <= 20; lag++)
+        {
+            if (n - lag <= 1)
+                continue;
+
+            double[] diffs = new double[n - lag];
+            for (int i = 0; i < diffs.Length; i++)
+                diffs[i] = spreadMids[i + lag] - spreadMids[i];
+
+            double stddev = diffs.StandardDeviation();
+
+            if (stddev > 0)
+            {
+                lags.Add(Math.Log(lag));
+                tau.Add(Math.Log(stddev));
+            }
+        }
+
+        if (lags.Count < 2)
+            return double.NaN;
+
+        var (intercept, slope) = SimpleRegression.Fit(lags.ToArray(), tau.ToArray());
+        double hurst = slope * 0.5 + 0.5;
+        return hurst;
+    }
+
+    /// <summary>
+/// Computes the mean and standard deviation of the spread mid-prices
+/// over a trailing, causal window.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Tuple of (Mean, StdDev). Returns (NaN, NaN) if input is too short.</returns>
+public static (double Mean, double StdDev) EvaluateMeanAndStdDev(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    return (mean, stddev);
+}
+
+
+   /// <summary>
+   /// Counts how many times the spread ask dropped below (mean - threshold) [buy signal]
+   /// and how many times the spread bid rose above (mean + threshold) [sell signal].
+   /// Assumes spreadData is a trailing, causal window.
+   /// </summary>
+   /// <returns>(BuyExcursions, SellExcursions)</returns>
+   public static (uint BuyExcursions, double cumm_buy_price, uint SellExcursions, double cumm_sell_price) EvaluateExcursionCount(
+       List<Tuple<double, double, double, double>> spreadData,
+       double entryThresholdMultiplier = 0.5) {
+      if (spreadData == null || spreadData.Count < 2)
+         return (0, Double.NaN, 0, Double.NaN);
+
+      var midPrices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      double mean = midPrices.Average();
+      double stddev = midPrices.StandardDeviation();
+      double buy_threshold = mean - entryThresholdMultiplier * stddev;
+      double sell_threshold = mean + entryThresholdMultiplier * stddev;
+
+      int buyCount = 0;
+      int sellCount = 0;
+      //bool in_buy_zone = false;
+      //bool in_sell_zone = false;
+
+      double cumm_buy_price = 0.0, cumm_sell_price = 0.0;
+      foreach (var (bid, _, ask, _) in spreadData) {
+         if (ask <= buy_threshold) {
+            //if (!in_buy_zone) {
+            cumm_buy_price += ask;
+            buyCount++;
+            //in_buy_zone = true;
+         }
+         //} else {
+         //   in_buy_zone = false;
+         //}
+
+         if (bid >= sell_threshold) {
+            //if (!in_sell_zone) {
+            cumm_sell_price += bid;
+            sellCount++;
+            //in_sell_zone = true;
+         }
+         //} else {
+         //   in_sell_zone = false;
+         //}
+      }
+
+      return ((uint)buyCount, cumm_buy_price / buyCount, (uint)sellCount, cumm_sell_price / sellCount);
+   }
+
+/// <summary>
+/// Counts how many times the spread mid-price crosses its mean level.
+/// Each crossing indicates a potential oscillation turn (cycle midpoint).
+/// Assumes the input is a trailing, causal window of spread ticks.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <returns>Number of zero crossings in the mid-price series</returns>
+public static UInt32 EvaluateZeroCrossingCount(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count < 2)
+        return 0;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    int zeroCrossings = 0;
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        double prevOffset = mids[i - 1] - mean;
+        double currOffset = mids[i] - mean;
+
+        if (prevOffset * currOffset < 0) // Sign changed
+            zeroCrossings++;
+    }
+
+    return (UInt32)zeroCrossings;
+}
+
+   /// <summary>
+   /// Computes the half-life of mean reversion.
+   /// Lower half-life means the spread reverts to its mean more quickly.
+   /// </summary>
+   public static double ComputeHalfLife(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var lagged = prices.Take(prices.Count - 1).ToArray();
+      var delta = prices.Skip(1).Zip(lagged, (cur, prev) => cur - prev).ToArray();
+
+      if (lagged.Length < 2) return double.NaN;
+
+      var x = Vector<double>.Build.Dense(lagged);
+      var y = Vector<double>.Build.Dense(delta);
+      var beta = x.DotProduct(y) / x.DotProduct(x);
+
+      return -Math.Log(2) / beta;
+   }
+
+   /// <summary>
+   /// Computes the lag-1 autocorrelation of the spread.
+   /// Lower values indicate quicker mean-reversion.
+   /// </summary>
+   public static double ComputeLag1Autocorrelation(List<Tuple<double, double, double, double>> spreadData) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      var mean = prices.Average();
+      var numerator = 0.0;
+      var denominator = 0.0;
+
+      for (int i = 1; i < prices.Count; i++) {
+         numerator += (prices[i] - mean) * (prices[i - 1] - mean);
+         denominator += Math.Pow(prices[i - 1] - mean, 2);
+      }
+
+      return denominator == 0 ? 0 : numerator / denominator;
+   }
+
+   /// <summary>
+   /// Estimates how quickly extreme Z-score values decay back toward the mean.
+   /// Shorter average decay time indicates stronger mean-reversion.
+   /// </summary>
+   public static double ComputeAverageZScoreDecay(List<Tuple<double, double, double, double>> spreadData, int zScoreWindow = 50) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < zScoreWindow + 2) return double.NaN;
+
+      var decayTimes = new List<int>();
+      for (int i = zScoreWindow; i < prices.Count - 1; i++) {
+         var window = prices.Skip(i - zScoreWindow).Take(zScoreWindow).ToList();
+         double mean = window.Average();
+         double stdDev = Statistics.StandardDeviation(window);
+         if (stdDev == 0) continue;
+
+         double z = (prices[i] - mean) / stdDev;
+         if (Math.Abs(z) > 2) {
+            for (int j = 1; j < prices.Count - i; j++) {
+               double zNext = (prices[i + j] - mean) / stdDev;
+               if (Math.Abs(zNext) < 1) {
+                  decayTimes.Add(j);
+                  break;
+               }
+            }
+         }
+      }
+
+      return decayTimes.Count == 0 ? double.NaN : decayTimes.Average();
+   }
+
+   /// <summary>
+   /// Computes the variance ratio: short-term variance divided by long-term variance.
+   /// Lower ratio suggests stronger mean-reversion.
+   /// </summary>
+   public static double ComputeVarianceRatio(List<Tuple<double, double, double, double>> spreadData, int shortWindow, int longWindow) {
+      var prices = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+      if (prices.Count < longWindow) return double.NaN;
+
+      double shortVar = Statistics.Variance(prices.Skip(prices.Count - shortWindow).Take(shortWindow));
+      double longVar = Statistics.Variance(prices.Skip(prices.Count - longWindow).Take(longWindow));
+
+      return longVar == 0 ? double.NaN : shortVar / longVar;
+   }
+
+   /// <summary>
+   /// Computes the highest bid and the lowest ask from a spread tick series.
+   /// Useful for evaluating the best market opportunity range.
+   /// </summary>
+   /// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+   /// <returns>Tuple of (MaxBid, MinAsk)</returns>
+   public static (double MaxBid, double MinAsk) EvaluateMaxBidMinAsk(List<Tuple<double, double, double, double>> spreadData)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return (double.NaN, double.NaN);
+
+    double maxBid = double.NegativeInfinity;
+    double minAsk = double.PositiveInfinity;
+
+    foreach (var (bid, _, ask, _) in spreadData)
+    {
+        if (bid > maxBid) maxBid = bid;
+        if (ask < minAsk) minAsk = ask;
+    }
+
+    return (maxBid, minAsk);
+}
+
+/// <summary>
+/// Computes the distribution of spread mid-prices around the mean,
+/// grouped by caller-defined distance bands (in standard deviations).
+///
+/// This helps answer:
+/// - Is the spread tightly hugging the mean?
+/// - Are there wide excursions or heavy tails?
+///
+/// You define breakpoints like: [0.5, 1.0, 2.0] → it will return:
+/// - "≤ 0.5σ"
+/// - "0.5σ – 1.0σ"
+/// - "1.0σ – 2.0σ"
+/// - "> 2.0σ"
+///
+/// How to Interpret It:
+/// - ≤ 0.5σ        → Tight clustering near the mean (strong mean-reversion)
+/// - 0.5σ – 1.0σ   → Mild oscillation (common trade region)
+/// - 1.0σ – 2.0σ   → Moderate deviation (potential trade signals)
+/// - > 2.0σ        → Rare, extreme moves (tail events, potential entries/exits)
+///
+/// Use this to tune entry thresholds or detect regime shifts.
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="regions">List of positive breakpoints in std deviations (e.g., [0.5, 1.0, 2.0])</param>
+/// <returns>Dictionary mapping label to % of mid-prices in that region</returns>
+public static Dictionary<string, double> EvaluateDistributionAroundMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    List<double> regions)
+{
+    var result = new Dictionary<string, double>();
+
+    if (spreadData == null || spreadData.Count < 2 || regions == null || regions.Count == 0)
+        return result;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+    double mean = mids.Average();
+    double stddev = Math.Sqrt(mids.Select(x => Math.Pow(x - mean, 2)).Average());
+
+    if (stddev == 0)
+    {
+        result["Zero Volatility"] = 100.0;
+        return result;
+    }
+
+    var buckets = new int[regions.Count + 1];
+    foreach (var mid in mids)
+    {
+        double distance = Math.Abs(mid - mean);
+        bool placed = false;
+
+        for (int i = 0; i < regions.Count; i++)
+        {
+            if (distance <= regions[i])
+            {
+                buckets[i]++;
+                placed = true;
+                break;
+            }
+        }
+
+        if (!placed)
+            buckets[regions.Count]++;
+    }
+
+    int total = mids.Count;
+    for (int i = 0; i < regions.Count; i++)
+    {
+        string label = i == 0
+            ? $"≤ {regions[i]:0.##}σ"
+            : $"{regions[i - 1]:0.##}σ – {regions[i]:0.##}σ";
+        result[label] = 100.0 * buckets[i] / total;
+    }
+
+    result[$"> {regions.Last():0.##}σ"] = 100.0 * buckets.Last() / total;
+
+    return result;
+}
+
+/// <summary>
+/// Computes two drift metrics indicating how much the spread's mean changes over time,
+/// based on the standard deviation of rolling mean values.
+///
+/// ✅ Absolute Drift: Standard deviation of the rolling mean
+/// ✅ Relative Drift: Drift normalized by the overall mean (unitless ratio)
+///
+/// This shows whether the center of the spread is stable — critical for mean-reversion.
+///
+/// How to Interpret the Output:
+/// - ✅ Relative Drift < 1% → Very stable mean — ideal
+/// - ⚠️ 1–5% → Mild drift — consider shorter holds or adaptive mean
+/// - ❌ >5% → Drifting mean — fixed mean may no longer work
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="window">Window size for rolling mean (e.g., 20 ticks)</param>
+/// <returns>
+/// Tuple of (Absolute Drift, Relative Drift) — where:
+/// - Absolute Drift = StdDev of rolling means
+/// - Relative Drift = Drift / global mean
+/// </returns>
+public static (double AbsoluteDrift, double RelativeDrift) EvaluateMeanDriftScore(
+    List<Tuple<double, double, double, double>> spreadData,
+    int window = 20)
+{
+    if (spreadData == null || spreadData.Count < window)
+        return (double.NaN, double.NaN);
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToArray();
+    var rollingMeans = new List<double>();
+
+    for (int i = window; i <= mids.Length; i++)
+    {
+        double rollingMean = mids.Skip(i - window).Take(window).Average();
+        rollingMeans.Add(rollingMean);
+    }
+
+
+    double overallMean = mids.Average(); // use full mean for normalization
+    //double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+    double drift = Math.Sqrt(rollingMeans.Select(m => Math.Pow(m - rollingMeans.Average(), 2)).Average());
+
+
+double relativeDrift = Math.Abs(overallMean) > 1e-8 ? drift / Math.Abs(overallMean) : double.NaN;
+    return (drift, relativeDrift);
+}
+
+/// <summary>
+/// Returns the latest Kalman Filter estimate of the true mean of the spread mid-prices.
+/// This tracks the spread's central value while allowing for gradual drift and suppressing noise.
+///
+/// Internally, this applies a recursive 1D Kalman filter over the input mid-prices.
+/// 
+/// Tuning Parameters:
+/// - Q: Process noise variance (higher = faster adaptation to drift)
+/// - R: Measurement noise variance (higher = more smoothing)
+/// 
+/// Use this instead of SMA if your mean drifts or if price series is noisy.
+///
+/// </summary>
+/// <param name="spreadData">List of (bid, bidQty, ask, askQty)</param>
+/// <param name="Q">Process noise variance (default = 0.0001)</param>
+/// <param name="R">Measurement noise variance (default = 0.01)</param>
+/// <returns>Latest estimated Kalman-filtered mean, or NaN if input is invalid</returns>
+public static double GetLatestKalmanMean(
+    List<Tuple<double, double, double, double>> spreadData,
+    double Q = 0.0001,
+    double R = 0.01)
+{
+    if (spreadData == null || spreadData.Count == 0)
+        return double.NaN;
+
+    var mids = spreadData.Select(x => (x.Item1 + x.Item3) / 2.0).ToList();
+
+    double x = mids[0];  // initial estimate of mean
+    double P = 1.0;       // initial uncertainty
+
+    for (int i = 1; i < mids.Count; i++)
+    {
+        // Predict step
+        double x_prior = x;
+        double P_prior = P + Q;
+
+        // Update step
+        double K = P_prior / (P_prior + R);
+        x = x_prior + K * (mids[i] - x_prior);
+        P = (1 - K) * P_prior;
+    }
+
+    return x; // the latest estimated mean
+}
+
+
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs

@@ -0,0 +1,137 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using System.Diagnostics;
+
+   public class SpreadPerformanceStream : ViewStream {
+      ViewStream _spread_price_stream = null;
+      private readonly IBlueTableSchema _schema;
+      private readonly int _window_secs;
+      private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
+      private ViewStream _bbo_stream;
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                       double buy_threshold, double sell_threshold, double flat_threshold)
+      : this(s1, s2, w1, w2, window_secs, buy_threshold, sell_threshold, flat_threshold, ViewStreamDatabase.instance) { }
+
+      public SpreadPerformanceStream(string s1, string s2, double w1, double w2, int window_secs,
+                                      double buy_threshold, double sell_threshold, double flat_threshold,
+                                      ViewStreamDatabase vsd) {
+         _bbo_stream = new BBOStream(vsd);
+         _schema = BlueDataUtility.form_schema("SpreadPerformance", Tuple.Create("id", TypeCode.Int32),                                                
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                // means / std dev                                          
+                                                Tuple.Create("stddev", TypeCode.Double),
+                                                Tuple.Create("mean", TypeCode.Double),
+                                                Tuple.Create("kalman_filter_mean", TypeCode.Double),
+                                                Tuple.Create("absolute_mean_drift", TypeCode.Double),
+                                                Tuple.Create("relative_mean_drift", TypeCode.Double),
+                                                // statistical measures
+                                                Tuple.Create("hurst_exponent", TypeCode.Double),
+                                                Tuple.Create("half_life", TypeCode.Double),
+                                                Tuple.Create("lag1_autocorrelation", TypeCode.Double),
+                                                Tuple.Create("average_zscore_decay", TypeCode.Double),
+                                                Tuple.Create("variance_ratio", TypeCode.Double),
+                                                Tuple.Create("n_zero_crossings", TypeCode.UInt32),                                                
+                                                // distributions
+                                                Tuple.Create("Z ≤ 0.5σ", TypeCode.Double),
+                                                Tuple.Create("0.5σ – 1.0σ", TypeCode.Double),
+                                                Tuple.Create("1.0σ – 2.0σ", TypeCode.Double),
+                                                Tuple.Create("> 2.0σ", TypeCode.Double),
+                                                ////////////////////////////////////////                                                   
+                                                Tuple.Create("max_bid", TypeCode.Double),
+                                                Tuple.Create("min_ask", TypeCode.Double),
+                                                Tuple.Create("n_buy_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_buy_price", TypeCode.Double),
+                                                Tuple.Create("n_sell_signals", TypeCode.UInt32),
+                                                Tuple.Create("cumm_sell_price", TypeCode.Double)
+                                                //Tuple.Create("pnl", TypeCode.Double),
+                                                //Tuple.Create("buy_turns", TypeCode.UInt32),
+                                                //Tuple.Create("sell_turns", TypeCode.UInt32),
+                                                //Tuple.Create("trades", TypeCode.String)
+                                                );
+         _window_secs = window_secs;
+         _buy_threshold = buy_threshold;
+         _sell_threshold = sell_threshold;
+         _flat_threshold = flat_threshold;
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, _bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, TimeSpan.FromSeconds(5.0), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
+         DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
+         var sw = Stopwatch.StartNew();
+         sw.Start();
+         foreach (IBlueRow r in dp.newRows) {
+            DateTime ts = r.get_datetime("timestamp");
+            TimeSpan elapsed_time = DateTime.Now - ts;
+            if (elapsed_time.TotalSeconds >= _window_secs)
+               continue;
+            spread_ts_prices.Add(Tuple.Create(
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz")));
+            if (ts <= min_time) {
+               min_time = ts;
+            }
+            if (ts >= max_time) {
+               max_time = ts;
+            }
+         }
+         List<Tuple<double, double, double, double>> spread_prices = spread_ts_prices
+                                                                                             .OrderBy(t => t.Item1)
+                                                                                             .Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5))
+                                                                                             .ToList();
+
+         uint n_ticks = (uint)spread_prices.Count;
+         double curr_windows_seconds = (max_time - min_time).TotalSeconds;
+         double hurst_exponent = SpreadPerformanceEvaluator.EvaluateHurstExponent(spread_prices);
+         double half_life = SpreadPerformanceEvaluator.ComputeHalfLife (spread_prices);
+         double lag1_autocorrelation = SpreadPerformanceEvaluator.ComputeLag1Autocorrelation(spread_prices);
+         double average_zscore_decay = SpreadPerformanceEvaluator.ComputeAverageZScoreDecay(spread_prices);
+         double variance_ratio = SpreadPerformanceEvaluator.ComputeVarianceRatio(spread_prices, 60, 360);
+
+         (double mean, double stddev) = SpreadPerformanceEvaluator.EvaluateMeanAndStdDev(spread_prices);
+         double kalman_filter_mean = SpreadPerformanceEvaluator.GetLatestKalmanMean(spread_prices);
+         uint n_zero_crossings = SpreadPerformanceEvaluator.EvaluateZeroCrossingCount(spread_prices);
+         (double max_bid, double min_ask) = SpreadPerformanceEvaluator.EvaluateMaxBidMinAsk(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices, _buy_threshold);
+         //(double pnl, List<SpreadPerformanceEvaluator.TradeAction> trades, int buy_turns, int sell_turns) = SpreadPerformanceEvaluator.EvaluateSimulatedPnL(spread_prices, 30, _buy_threshold, _flat_threshold);
+         //string trade_str = String.Empty;
+         //foreach (var tr in trades) {
+         //   trade_str += tr.ToString() + " , ";
+         //}
+         Dictionary<string, double> dist_map = SpreadPerformanceEvaluator.EvaluateDistributionAroundMean(spread_prices,
+                                                                       new List<double>(new double[] { 0.5, 1.0, 2.0 }));
+         var dist_vals = dist_map.Count < 4 ? new double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN } : dist_map.Values.ToArray();
+         (double absolute_drift, double relative_drift) = SpreadPerformanceEvaluator.EvaluateMeanDriftScore(spread_prices);
+         IBlueRow nr = new BlueRow(_schema, 0, min_time, max_time, n_ticks, curr_windows_seconds, stddev, mean, kalman_filter_mean, absolute_drift, relative_drift,
+            hurst_exponent, half_life, lag1_autocorrelation, average_zscore_decay, variance_ratio, n_zero_crossings,
+                                    dist_vals[0], dist_vals[1], dist_vals[2], dist_vals[3],
+                                    max_bid, min_ask, n_buy_signals, cumm_buy_price, n_sell_signals, cumm_sell_price);
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(t, myLastValue as DataPacket, true);
+         var perf_msecs = sw.ElapsedMilliseconds;
+         sw.Stop();
+         Log.Gen.info($"SpreadPerformanceStream._handle_spread_prices comp_time: {perf_msecs} msecs");
+         myFireEvent(dp_out);
+
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs

@@ -0,0 +1,68 @@
+namespace bts.SilverPine{
+   using BCG;
+   public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
+      private IBlueTableSchema _schema;
+      private ViewStream _leg_price_stream;
+      private readonly double _window_secs;
+      private readonly string _s1, _s2;
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs)
+         : this(s1, s2, window_secs, sample_period_secs, ViewStreamDatabase.instance) { }
+
+
+      public SpreadStatArbsWeightLinearRegressionStream(string s1, string s2, double window_secs, double sample_period_secs, ViewStreamDatabase vsd) {
+         _window_secs = window_secs;
+         _s1 = s1;
+         _s2 = s2;
+         _schema = BlueDataUtility.form_schema("LinearRegressionWeights",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("t0", TypeCode.DateTime),
+                                                Tuple.Create("t1", TypeCode.DateTime),
+                                                Tuple.Create("curr_windows_seconds", TypeCode.Double),
+                                                Tuple.Create("n_ticks", TypeCode.UInt32),
+                                                Tuple.Create("s1", TypeCode.String),
+                                                Tuple.Create("w1", TypeCode.Double),
+                                                Tuple.Create("s2", TypeCode.String),
+                                                Tuple.Create("w2", TypeCode.String));
+
+         _leg_price_stream = new TimeSeriesStream(new SyntheticSpreadLegsStream(s1, s2));
+         _leg_price_stream = new ThrottleStream(_leg_price_stream, TimeSpan.FromSeconds(1), vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _leg_price_stream.onChange += _handle_leg_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _leg_price_stream.onChange -= _handle_leg_prices;
+      }
+
+      private void _handle_leg_prices(object dp_obj){
+         DataPacket dp = dp_obj as DataPacket;
+         List<Tuple<double, double, DateTime>> leg_prices = new List<Tuple<double, double, DateTime>>();
+         foreach (IBlueRow r in dp.newRows){
+            double s1_mid = (r.get<double>("s1_bid_price") + r.get<double>("s1_ask_price")) /2.0;
+            double s2_mid = (r.get<double>("s2_bid_price") + r.get<double>("s2_ask_price")) /2.0;
+            DateTime ts = r.get_datetime("timestamp");
+            leg_prices.Add(new Tuple<double, double, DateTime>(s1_mid, s2_mid, ts));
+         }
+         if (leg_prices.Count <= 2)
+            return;
+         leg_prices = leg_prices.Where(t => (DateTime.Now - t.Item3).TotalSeconds <= _window_secs).ToList();
+         DateTime t0 = leg_prices.Select(t => t.Item3).Min();
+         DateTime t1 = leg_prices.Select(t => t.Item3).Max();
+         double curr_windows_seconds = (t1 - t0).TotalSeconds;
+         int n_ticks = leg_prices.Count;
+         List<double> s1_mids = leg_prices.Select(t => t.Item1).ToList();
+         List<double> s2_mids = leg_prices.Select(t => t.Item2).ToList();
+         (double w1, double w2) = SpreadWeightCalculator.CalculateWeightsLinearRegression(s1_mids, s2_mids);
+         BlueTable state = new BlueTable(_schema);
+         IBlueRow nr = new BlueRow(_schema, 0, t0, t1, curr_windows_seconds, n_ticks,_s1, w1, _s2, w2);
+         state.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+
+      }
+   }
+}
+

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs

@@ -0,0 +1,143 @@
+namespace bts.SilverPine {
+   using BCG;
+
+   using MathNet.Numerics.LinearAlgebra.Factorization;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Diagnostics;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("SpreadStatsStream")]
+   // This is the fundamental stream that computes the stats (including buy and sell zones) 
+   // of a spread. the legs and the weights are passed as args
+   public class SpreadStatsStream : ViewStream {
+      private ViewStream _spread_price_stream;
+      private readonly IBlueTableSchema _schema;
+      private readonly TimeSpan _window_sz;
+      private readonly string _spread;
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs)
+         : this(s1, s2, w1, w2, rolling_window_sz_seconds, window_sampling_sz_secs, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) { }
+      public SpreadStatsStream(string s1, string s2, double w1, double w2, int rolling_window_sz_seconds, int window_sampling_sz_secs, ViewStream bbo_stream, ViewStreamDatabase vsd) {
+         _window_sz = TimeSpan.FromSeconds(rolling_window_sz_seconds);
+         _spread = $"{w1} X {s1} - {w2} X {s2}";
+         _schema = new BlueTableSchema("SpreaddStats", new ColumnInfo ("id", 0, TypeCode.Int32));
+         _schema = _schema.add_column(TypeCode.String, "spread");
+         _schema = _schema.add_column(TypeCode.String, "window");
+         // basic stats
+         _schema = _schema.add_column(TypeCode.Double, "bid_mean");
+         _schema = _schema.add_column(TypeCode.Double, "ask_mean");
+         _schema = _schema.add_column(TypeCode.Double, "bid_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "ask_stddev");
+         _schema = _schema.add_column(TypeCode.Double, "bid_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "ask_skewness");
+         _schema = _schema.add_column(TypeCode.Double, "bid_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "ask_kurtosis");
+         _schema = _schema.add_column(TypeCode.Double, "bid_max");
+         _schema = _schema.add_column(TypeCode.Double, "ask_max");
+         _schema = _schema.add_column(TypeCode.Double, "bid_min");
+         _schema = _schema.add_column(TypeCode.Double, "ask_min");
+         // bid bins
+         _schema = _schema.add_column(TypeCode.Double, "bid Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "bid 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "bid Z ≥ 2σ");
+         // ask bins
+         _schema = _schema.add_column(TypeCode.Double, "ask Z < -2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -2σ ≤ Z < -1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask -1σ ≤ Z < 0σ");
+         // positive z's
+         _schema = _schema.add_column(TypeCode.Double, "ask 0σ ≤ Z < 1σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask 1σ ≤ Z < 2σ");
+         _schema = _schema.add_column(TypeCode.Double, "ask Z ≥ 2σ");
+         // opps
+         _schema = _schema.add_column(TypeCode.Int32, "n_buy_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_buy_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_sell_ticks");
+         _schema = _schema.add_column(TypeCode.Double, "cumm_sell_price");
+         _schema = _schema.add_column(TypeCode.Int32, "n_trade_ticks");
+         _spread_price_stream = new SyntheticSpreadStream(s1, s2, w1, w2, bbo_stream, vsd);
+         _spread_price_stream = new TimeSeriesStream(_spread_price_stream);
+         double window_sampling_sz_smsecs = window_sampling_sz_secs * 1000.0 + new Random(DateTime.Now.Second).Next(-100, 100);
+         TimeSpan sampling_period = TimeSpan.FromMilliseconds(window_sampling_sz_smsecs);
+         _spread_price_stream = new ThrottleStream(_spread_price_stream, sampling_period, vsd);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _spread_price_stream.onChange += _handle_spread_prices;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _spread_price_stream.onChange -= _handle_spread_prices;
+      }
+
+      private void _handle_spread_prices(object dp_obj) {
+         List<IBlueRow> rows = (dp_obj as DataPacket).newRows.to_list();
+         Func<IBlueRow, bool> row_filter = (r) => {
+            DateTime ts = r.get_datetime("timestamp");
+            DateTime start_time = DateTime.Now.Subtract(_window_sz);
+            return ts >= start_time;
+         };
+         var rolling_win_rows = rows.Where(r => row_filter(r));
+         var spread_prices = rolling_win_rows.Select(r => (
+                                          r.get_datetime("timestamp"),
+                                          r.get<double>("spread_bid_price"),
+                                          r.get<double>("spread_bid_sz"),
+                                          r.get<double>("spread_ask_price"),
+                                          r.get<double>("spread_ask_sz"))).ToList();
+         spread_prices.Sort((a,b) => a.Item1.CompareTo(b.Item1));
+         DateTime t0 = spread_prices.First().Item1, t1 = spread_prices.Last().Item1;
+         var tot_secs = (t1 - t0).TotalSeconds;
+         var stopwatch = Stopwatch.StartNew();
+         Log.Gen.info($"SpreadStatsStream.analyze_spread_quotes executed in {stopwatch.ElapsedMilliseconds} ms");
+         var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices);
+         (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = 
+              SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0);
+         IBlueRow nr = new BlueRow(_schema,
+            0,
+            _spread,
+            $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs",
+            result.bid_stats.Mean,
+            result.ask_stats.Mean,
+            result.bid_stats.StandardDeviation,
+            result.ask_stats.StandardDeviation,
+            result.bid_stats.Skewness,
+            result.ask_stats.Skewness,
+            result.bid_stats.Kurtosis,
+            result.ask_stats.Kurtosis,
+            result.bid_stats.Maximum,
+            result.ask_stats.Maximum,
+            result.bid_stats.Minimum,
+            result.ask_stats.Minimum,
+            // bid distribution
+            result.bid_zscore_distribution["Z < -2σ"],
+            result.bid_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.bid_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.bid_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.bid_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.bid_zscore_distribution["Z ≥ 2σ"],
+            // ask distributions
+            result.ask_zscore_distribution["Z < -2σ"],
+            result.ask_zscore_distribution["-2σ ≤ Z < -1σ"],
+            result.ask_zscore_distribution["-1σ ≤ Z < 0σ"],
+            result.ask_zscore_distribution["0σ ≤ Z < 1σ"],
+            result.ask_zscore_distribution["1σ ≤ Z < 2σ"],
+            result.ask_zscore_distribution["Z ≥ 2σ"],
+            n_buy_signals,
+            cumm_buy_price,
+            n_sell_signals,
+            cumm_sell_price,
+            n_buy_signals + n_sell_signals
+            );
+         stopwatch.Stop();
+         BlueTable t = new BlueTable(_schema);
+         t.addRow(nr);
+         var dp_out = BlueDataUtility.build_dp(t, myLastValue as DataPacket);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using MathNet.Numerics.LinearRegression;
+
+namespace bts.SilverPine {
+
+   public static class SpreadWeightCalculator {
+      /// <summary>
+      /// Estimates hedge weights (w1, w2) such that the spread s = w1 * s1 - w2 * s2 
+      /// is potentially mean-reverting. Uses linear regression: s1 = beta * s2 + ε.
+      /// 
+      /// Assumes mid-prices are precomputed and aligned over the same time window.
+      /// </summary>
+      /// <param name="s1MidPrices">List of mid-prices for asset 1 (dependent variable)</param>
+      /// <param name="s2MidPrices">List of mid-prices for asset 2 (independent variable)</param>
+      /// <param name="includeIntercept">
+      /// Whether to include an intercept in the regression model (default: true).
+      /// </param>
+      /// <returns>(w1, w2): Tuple of hedge weights</returns>
+      public static (double w1, double w2) CalculateWeightsLinearRegression(
+          List<double> s1MidPrices,
+          List<double> s2MidPrices,
+          bool includeIntercept = true) {
+         if (s1MidPrices == null || s2MidPrices == null)
+            throw new ArgumentNullException("Price series cannot be null.");
+         if (s1MidPrices.Count != s2MidPrices.Count)
+            throw new ArgumentException("Price series must be of equal length.");
+         if (s1MidPrices.Count < 2)
+            throw new ArgumentException("Need at least two data points for regression.");
+
+         double[] x = s2MidPrices.ToArray(); // Independent variable
+         double[] y = s1MidPrices.ToArray(); // Dependent variable
+
+         double beta;
+
+         if (includeIntercept) {
+            // Returns (intercept, slope)
+            (double intercept, double slope) = SimpleRegression.Fit(x, y);
+            beta = slope;
+         } else {
+            // Returns only slope when forcing regression through origin
+            beta = SimpleRegression.FitThroughOrigin(x, y);
+         }
+
+         // Return fixed w1 = 1.0 and w2 = beta
+         return (1.0, beta);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/VWAPStream.cs

@@ -0,0 +1,118 @@
+// TODO: Remove
+//// namespace bts.SilverPine 
+////    using bts;
+////    using BCG;
+////    using MongoDB.Driver.Linq;
+////    using System.ComponentModel.DataAnnotations;
+////    using ZstdSharp.Unsafe;
+
+////    [DataStreamUserRespresentation("VWAP")]
+////    public class VWAPStream : ViewStream {
+////       // returns the VWAP stream for each contract 
+////       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
+////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+////          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
+////          // stream is grouped per contract and then vwap is calculated per contract
+////          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
+////          (rows, idx) => {
+////             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
+////             IBlueRow ir = rows.ElementAt((int)idx);
+////             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
+////             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
+////             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
+////             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
+////             MutableBlueRow nr = new MutableBlueRow(schema, ir);
+////             //vwap = -313.987;
+////             nr.setValue("vwap", vwap);
+////             return nr;
+////          });
+////          return rs;
+////       }
+////       // the args want dt1 > dt2, if its nto we, swap them
+////       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+////          if (dt1 < dt2) {
+////             DateTime aux = dt1;
+////             dt1 = dt2;
+////             dt2 = aux;
+////          }
+////       }
+
+////       // this assumes the ohlc_str has diff contracts
+////       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
+////          swap_if_necessary(ref begin_dt, ref end_dt);
+////          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
+////          string symbol = String.Empty;
+////          symbol = ccat_sym_contr.Item2;
+////          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
+////          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
+////          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
+////          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
+////          ViewStream rs = vwap_str;
+////          switch (ccat_sym_contr.Item1) {
+////             case ContractUtils.ContractCategory.FutureString:
+////             case ContractUtils.ContractCategory.SpreadString:
+////                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
+////                break;
+////             case ContractUtils.ContractCategory.Symbol:
+////                rs = new FuturesActiveContractStream(vwap_str);
+////                break;
+////          }
+////          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
+////          rs = new NewProjectStream(rs, "! vwap");
+////          mySetRealSource(rs);
+////       }
+
+////       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
+////       :
+////       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
+////    }
+//// }
+
+//using System;
+
+//using BCG;
+
+//using bts.api;
+
+//namespace bts.SilverPine {
+//   /*
+//      ----------------------------------------------------------------------
+//                                  *Expression*
+//      ----------------------------------------------------------------------
+//          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
+
+//          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
+//          r=math(r, "vprice = volume * close")
+//          r=aggregate(r, "vprice:Sum, volume:Sum")
+//          r=math(r, "vwap = vprice_sum / volume_sum")
+//          r=newProject(r, "vwap !")
+
+//      ----------------------------------------------------------------------
+//                                    *Schema*
+//      ----------------------------------------------------------------------
+//          id                              System.Int32
+//          vwap                            System.Double
+//   */
+
+//   [DataStreamUserRespresentation("VWAP")]
+//   public class VWAPStream : ViewStream {
+//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
+//      :
+//      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
+
+//      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
+//         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
+//         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
+//         // since this is already filtered, we can groupBy iit without a groupBy column
+//         r = new MathStream(r,
+//                 "vprice = volume * close");
+//         r = new NonIncGroupByStream(r,
+//                                      "vprice:Sum, volume:Sum");
+//         r = new MathStream(r,
+//                             "vwap = vprice_sum / volume_sum");
+//         r = new NewProjectStream(r,
+//                                   "vwap !");
+//         mySetRealSource(r);
+//      }
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -111,6 +111,7 @@ namespace bts.SilverPine {
                                                 Tuple.Create("id", TypeCode.Int32),
                                                 Tuple.Create("sec_key", TypeCode.UInt32),
                                                 Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("exchange", TypeCode.String),
                                                 Tuple.Create("bid_price", TypeCode.Double),
                                                 Tuple.Create("bid_sz", TypeCode.Single),
                                                 Tuple.Create("ask_price", TypeCode.Double),
@@ -122,14 +123,16 @@ namespace bts.SilverPine {
       }
 
       public BBOStream(ViewStreamDatabase vsd) {
-         // bts.Log.Gen.info("BBOStream.cons");
-         // _vsd = vsd;
-         // _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-         FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
-         mySetRealSource(rs);
+         bts.Log.Gen.info("BBOStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+         //FakeBBOStream rs = new FakeBBOStream("Future", "CME", "ES", 5000, 0.25, vsd);
+         //mySetRealSource(rs);
       }
 
       protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.books().Subscribe(_on_books);
          _vsd?.addWorker(this);
          _md_client?.books().Subscribe(_on_books);
       }
@@ -141,14 +144,14 @@ namespace bts.SilverPine {
          }
          foreach (Change<BookTuple> c in update.changes) {
             var bt = c.New;
-            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+            //bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
          }
       }
 
       private void _on_books(TableUpdate<BookTuple, uint> update) {
          _log_updates(update);
          // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         //bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
          _cached_state = update.state;
       }
 
@@ -158,7 +161,7 @@ namespace bts.SilverPine {
 
       private void _scan_bbo() {
          // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("BBOStream._scan_bbo");
+         //bts.Log.Gen.info("BBOStream._scan_bbo");
          if (_cached_state == null)
             return;
          BlueTable new_state = new BlueTable(_bbo_schema);
@@ -167,7 +170,8 @@ namespace bts.SilverPine {
             var sec = kv.Value.security;
             int id = Convert.ToInt32(skey);
             BookTuple bt = kv.Value;
-            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
+            var venue = sec.Exchange;
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, sec.ToString(), venue.ToString(), bt.book.bids[0].price, bt.book.bids[0].size,
                                      bt.book.asks[0].price, bt.book.asks[0].size);
             new_state.addRow(r);
          }

src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs

@@ -1,53 +1,53 @@
-using BCG;
+//using BCG;
 
-namespace bts.SilverPine {
-   /*
-      ----------------------------------------------------------------------
-                                  *Expression*
-      ----------------------------------------------------------------------
-          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
-          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
-          r=contractExpander(r,"symbol")
-          r=filter(r,"contract_type='FutureSpread'")
-          r=aggregate(r,"symbol","volume:Sum","ts_event")
-          r=aggregate(r,"volume_sum:Max")
-          r=newProject(r,"volume_sum_max:volume symbol !")
+//namespace bts.SilverPine {
+//   /*
+//      ----------------------------------------------------------------------
+//                                  *Expression*
+//      ----------------------------------------------------------------------
+//          //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM")
+//          r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025)
+//          r=contractExpander(r,"symbol")
+//          r=filter(r,"contract_type='FutureSpread'")
+//          r=aggregate(r,"symbol","volume:Sum","ts_event")
+//          r=aggregate(r,"volume_sum:Max")
+//          r=newProject(r,"volume_sum_max:volume symbol !")
 
-      ----------------------------------------------------------------------
-                                    *Schema*
-      ----------------------------------------------------------------------
-          symbol                          System.String
-          id                              System.Int32
-          volume                          System.Double
-   */
+//      ----------------------------------------------------------------------
+//                                    *Schema*
+//      ----------------------------------------------------------------------
+//          symbol                          System.String
+//          id                              System.Int32
+//          volume                          System.Double
+//   */
 
-   [DataStreamUserRespresentation("VolumeAgg")]
-   public class VolumeAggregationStream : ViewStream {
-      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
-      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
+//   [DataStreamUserRespresentation("VolumeAgg")]
+//   public class VolumeAggregationStream : ViewStream {
+//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end)
+//      : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { }
 
-      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
-      :
-      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
+//      public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end)
+//      :
+//      this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { }
 
-      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
-         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
-         ViewStream r;
-         r = new FutureHistoricalBarsStream(future_contract,
-                                             range_beg.ToString(), range_end.ToString());
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, $"contract_type='{contract_type}'");
-         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
-         r = new NonIncGroupByStream(r, "volume_sum:Max");
-         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
-         mySetRealSource(r);
-      }
-   }
+//      public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) {
+//         future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*";
+//         ViewStream r;
+//         r = new FutureHistoricalBarsStream(future_contract,
+//                                             range_beg.ToString(), range_end.ToString());
+//         r = new ContractExpanderStream(r, "symbol");
+//         r = new FilterStream(r, $"contract_type='{contract_type}'");
+//         r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event");
+//         r = new NonIncGroupByStream(r, "volume_sum:Max");
+//         r = new NewProjectStream(r, "volume_sum_max:volume symbol !");
+//         mySetRealSource(r);
+//      }
+//   }
 
-   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
-   public class PeriodicVolumeAggregationStream : ViewStream {
-      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
-                                             TimeSpan period, ViewStreamDatabase db) {
-      }
-   }
-}
+//   [DataStreamUserRespresentation("PeriodicVolumeAgg")]
+//   public class PeriodicVolumeAggregationStream : ViewStream {
+//      public PeriodicVolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end,
+//                                             TimeSpan period, ViewStreamDatabase db) {
+//      }
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs

@@ -1,117 +0,0 @@
-// namespace bts.SilverPine {
-//    using bts;
-//    using BCG;
-//    using MongoDB.Driver.Linq;
-//    using System.ComponentModel.DataAnnotations;
-//    using ZstdSharp.Unsafe;
-
-//    [DataStreamUserRespresentation("VWAP")]
-//    public class VWAPStream : ViewStream {
-//       // returns the VWAP stream for each contract 
-//       private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) {
-//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-//          ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params);
-//          // stream is grouped per contract and then vwap is calculated per contract
-//          GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract",
-//          (rows, idx) => {
-//             Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}");
-//             IBlueRow ir = rows.ElementAt((int)idx);
-//             var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt);
-//             var volumeprices = sel_rows.Select(r => Tuple.Create(r.get<int>("volume"), r.get<double>("close"))).ToList();
-//             double vwap = IndicatorCalc.calculate_VWAP(volumeprices);
-//             IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap"));
-//             MutableBlueRow nr = new MutableBlueRow(schema, ir);
-//             //vwap = -313.987;
-//             nr.setValue("vwap", vwap);
-//             return nr;
-//          });
-//          return rs;
-//       }
-//       // the args want dt1 > dt2, if its nto we, swap them
-//       static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-//          if (dt1 < dt2) {
-//             DateTime aux = dt1;
-//             dt1 = dt2;
-//             dt2 = aux;
-//          }
-//       }
-
-//       // this assumes the ohlc_str has diff contracts
-//       public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) {
-//          swap_if_necessary(ref begin_dt, ref end_dt);
-//          Tuple<ContractUtils.ContractCategory, string> ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract);
-//          string symbol = String.Empty;
-//          symbol = ccat_sym_contr.Item2;
-//          var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt);
-//          ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt);
-//          // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract,
-//          // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it
-//          ViewStream rs = vwap_str;
-//          switch (ccat_sym_contr.Item1) {
-//             case ContractUtils.ContractCategory.FutureString:
-//             case ContractUtils.ContractCategory.SpreadString:
-//                rs = new FilterStream(vwap_str, r => r.get<string>("contract").CompareTo(ccat_sym_contr.Item2) == 0);
-//                break;
-//             case ContractUtils.ContractCategory.Symbol:
-//                rs = new FuturesActiveContractStream(vwap_str);
-//                break;
-//          }
-//          rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0);
-//          rs = new NewProjectStream(rs, "! vwap");
-//          mySetRealSource(rs);
-//       }
-
-//       public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt)
-//       :
-//       this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { }
-//    }
-// }
-
-using System;
-
-using BCG;
-
-using bts.api;
-
-namespace bts.SilverPine {
-   /*
-      ----------------------------------------------------------------------
-                                  *Expression*
-      ----------------------------------------------------------------------
-          //nl("Find Settlement price of the SP500 future in the first week of 2023.")
-
-          r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024")
-          r=math(r, "vprice = volume * close")
-          r=aggregate(r, "vprice:Sum, volume:Sum")
-          r=math(r, "vwap = vprice_sum / volume_sum")
-          r=newProject(r, "vwap !")
-
-      ----------------------------------------------------------------------
-                                    *Schema*
-      ----------------------------------------------------------------------
-          id                              System.Int32
-          vwap                            System.Double
-   */
-
-   [DataStreamUserRespresentation("VWAP")]
-   public class VWAPStream : ViewStream {
-      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt)
-      :
-      this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { }
-
-      public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) {
-         //var fc = new ContractUtils.FutureContract(symbol_or_contract);
-         ViewStream r = new FutureHistoricalBarsStream(symbol_or_contract, beg_dt.ToString(), end_dt.ToString());
-         // since this is already filtered, we can groupBy iit without a groupBy column
-         r = new MathStream(r,
-                 "vprice = volume * close");
-         r = new NonIncGroupByStream(r,
-                                      "vprice:Sum, volume:Sum");
-         r = new MathStream(r,
-                             "vwap = vprice_sum / volume_sum");
-         r = new NewProjectStream(r,
-                                   "vwap !");
-         mySetRealSource(r);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -76,30 +76,3 @@ namespace bts.SilverPine {
       }
    }
 }
-
-
-namespace bts.SilverPine {
-   using BCG;
-
-   using bts.api;
-
-   using Newtonsoft.Json.Serialization;
-
-   [DataStreamUserRespresentation("FuturesContinousContract")]
-   public class FuturesContinousContractStream : ViewStream {
-      // the argument is symbol, like "ES" etc and not the contract itself
-      public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { }
-      public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) {
-         symbol = symbol.EndsWith("*") ? symbol : symbol + "*";
-         ViewStream r = new FuturesHistorical1DayBarsStream(symbol);
-         r = new ContractExpanderStream(r, "symbol");
-         r = new FilterStream(r, "contract_type='Future'");
-         // finding the max volume across symbols, if we need to do this for open_interest, then join here 
-         // with the open interest stream on symbol/contract
-         r = new NonIncGroupByStream(r, "ts_event", "volume:Max");
-         r = new NewProjectStream(r, "ts_event symbol !");
-         r = new SortStream(r, "-ts_event");
-         mySetRealSource(r);
-      }
-   }
-}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs

@@ -1,16 +1,18 @@
-namespace bts.SilverPine {
-   using bts;
-   using BCG;
 
-   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
-   // future exp becomes the highest volume one
-   [DataStreamUserRespresentation("FuturesRoll")]
-   public class FuturesRollStream : ViewStream {
-      public FuturesRollStream(string future_symbol) {
-         ViewStream rs = new FuturesContinousContractStream(future_symbol);
-         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
-         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
-         mySetRealSource(rs);
-      }
-   }
-}
+//TODO: Remove
+//namespace bts.SilverPine {
+//   using bts;
+//   using BCG;
+
+//   // it uses the volume (better to use open interest perhap) to detect the rollover, ie when a new 
+//   // future exp becomes the highest volume one
+//   [DataStreamUserRespresentation("FuturesRoll")]
+//   public class FuturesRollStream : ViewStream {
+//      public FuturesRollStream(string future_symbol) {
+//         ViewStream rs = new FuturesContinousContractStream(future_symbol);
+//         rs = new ColumnEvolutionStream(rs, "ts_event", "symbol");
+//         rs = new NewProjectStream(rs, "symbol ts_event:roll_date !");
+//         mySetRealSource(rs);
+//      }
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -1,8 +1,9 @@
-namespace BCG {
+namespace bts.SilverPine {
    using BCG.Utilities.General;
 
    using bts.api;
    using bts.md;
+   using BCG;
 
    using Spark.Core.Reactive;
 
@@ -31,6 +32,7 @@ namespace BCG {
                                                 Tuple.Create("sec_type", TypeCode.String),
                                                 Tuple.Create("venue", TypeCode.String),
                                                 Tuple.Create("symbol", TypeCode.String),
+                                                Tuple.Create("currency", TypeCode.String),
                                                 Tuple.Create("expiration", TypeCode.DateTime),
                                                 Tuple.Create("strike", TypeCode.Double),
                                                 Tuple.Create("callput", TypeCode.String)
@@ -81,7 +83,7 @@ namespace BCG {
          foreach (var kv in _cached_state) {
             SecurityKey skey = kv.Key;
             int id = Convert.ToInt32(skey);
-            SecurityImmutable sec = kv.Value;
+            SecurityImmutable sec = kv.Value;            
             bts.api.SecurityType sec_type = sec.security_type();
             object exp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
@@ -92,7 +94,7 @@ namespace BCG {
             object cp = DBNull.Value;
             if (sec_type == bts.api.SecurityType.Option)
                cp = sec.Option.CallPut.ToString();
-            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, exp, strike, cp);
             new_state.addRow(r);
          }
          DataPacket dp = null;
@@ -117,4 +119,4 @@ namespace BCG {
          }
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -174,10 +174,10 @@ namespace bts.SilverPine {
                // TODO: assuming 1 day only for historic               
                string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
                                                                                          //string table_name = $"{symbol}_{ohlcv_params.bar_size}";
-               string db_str = $"Data Source={_location}{_db_name};Version=3";
+               string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
                string sql_str = $"select * from {table_name}";
                // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
-               ViewStream rs = new SQLiteStream(db_str, sql_str);
+               ViewStream rs = new SQLiteStream(db_name, sql_str);
                rs = new SchemaGuesserStream(rs);
                if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
                   rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
@@ -207,4 +207,4 @@ namespace bts.SilverPine {
 
 
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs

@@ -1,26 +1,27 @@
-namespace bts.SilverPine {
-   using BCG;
+// TODO: Remove
+//namespace bts.SilverPine {
+//   using BCG;
 
-   using bts;
-   // used directly in blueorb
-   [DataStreamUserRespresentation("Settlement")]
-   public class SettlementPriceStream : ViewStream {
+//   using bts;
+//   // used directly in blueorb
+//   [DataStreamUserRespresentation("Settlement")]
+//   public class SettlementPriceStream : ViewStream {
 
-      // public SettlementPriceStream(string future_name, string date_txt)
-      // : this(future_name, date_txt, date_txt) { }
+//      // public SettlementPriceStream(string future_name, string date_txt)
+//      // : this(future_name, date_txt, date_txt) { }
 
-      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
-      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
+//      // public SettlementPriceStream(string future_name, string begin_dt, string end_dt)
+//      // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { }
 
-      public SettlementPriceStream(string future_contract, DateTime begin_dt)
-      : this(future_contract, begin_dt, begin_dt) { }
+//      public SettlementPriceStream(string future_contract, DateTime begin_dt)
+//      : this(future_contract, begin_dt, begin_dt) { }
 
-      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
-         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
-         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
-         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
-         mySetRealSource(rs);
-      }
+//      public SettlementPriceStream(string future_contract, DateTime begin_dt, DateTime end_dt) {
+//         ContractUtils.FutureContract fc = new ContractUtils.FutureContract(future_contract);
+//         ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt));
+//         rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !");
+//         mySetRealSource(rs);
+//      }
 
-   }
-}
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs

@@ -0,0 +1,125 @@
+using System;
+using BCG;
+
+namespace bts.SilverPine
+{
+/*
+   ----------------------------------------------------------------------
+                               *Expression*
+   ----------------------------------------------------------------------
+       s1 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s1 = addColumns(s1, "ljc:0")
+       s2 = filter(bbo(), "sec = 'ETH-USDT-SWAP'")
+       s2 = addColumns(s2, "rjc:0")
+       
+       s = fastJoin(s1, "ljc", "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz", 
+                    s2, "rjc", "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz")
+       
+
+   ----------------------------------------------------------------------
+                                 *Schema*
+   ----------------------------------------------------------------------
+       id                              System.Int32
+       s1                              System.String
+       s1_bid_price                    System.Double
+       s1_bid_sz                       System.Single
+       s1_ask_price                    System.Double
+       s1_ask_sz                       System.Single
+       s2                              System.String
+       s2_bid_price                    System.Double
+       s2_bid_sz                       System.Single
+       s2_ask_price                    System.Double
+       s2_ask_sz                       System.Single
+*/
+
+
+   // Combines the market data of spread legs into one stream, lined up nicely in a row.
+   // this is useful for operations like finding weights, that need leg market data organized together succintly.
+   public class SyntheticSpreadLegsStream : ViewStream 
+   {
+      public SyntheticSpreadLegsStream(string s1, string s2) : 
+         this(s1, s2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance) {}
+
+      public SyntheticSpreadLegsStream(string s1, string s2, ViewStream bbo_stream, ViewStreamDatabase vsb)
+      {
+         ViewStream vs1;
+         ViewStream vs2;
+         ViewStream rs;
+
+         vs1 = new FilterStream (new BBOStream (),
+                                $"sec = '{s1}'");
+         vs1 = new AddColumnsStream (vs1,
+                                    "ljc:0");
+         vs2 = new FilterStream (bbo_stream,
+                                $"sec = '{s2}'");
+         vs2 = new AddColumnsStream (vs2,
+                                    "rjc:0");
+         rs = new NonIncJoinStream (vs1,
+                                   "ljc",
+                                   "sec:s1 bid_price:s1_bid_price bid_sz:s1_bid_sz ask_price:s1_ask_price ask_sz:s1_ask_sz",
+                                   vs2,
+                                   "rjc",
+                                   "sec:s2 bid_price:s2_bid_price bid_sz:s2_bid_sz ask_price:s2_ask_price ask_sz:s2_ask_sz");
+         mySetRealSource(rs);
+      }
+   }
+   public class SyntheticSpreadStream : ViewStream{
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2)
+      : this(s1, s2, w1, w2, new BBOStream(ViewStreamDatabase.instance), ViewStreamDatabase.instance){}
+      public SyntheticSpreadStream (string s1, string s2, double w1, double w2, ViewStream bbo_stream, ViewStreamDatabase vsd){
+         ViewStream rs = new SyntheticSpreadLegsStream(s1, s2, bbo_stream, vsd);
+
+         Func<IBlueTableSchema, IBlueTableSchema> get_schema = (inp_schema) => {
+            IBlueTableSchema rv = inp_schema;
+            rv = rv.add_column(TypeCode.Double, "spread_bid_price");
+            rv = rv.add_column(TypeCode.Double, "spread_bid_sz");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_price");
+            rv = rv.add_column(TypeCode.Double, "spread_ask_sz");
+            return rv;
+         };
+
+         rs = new MapStream(rs, (inp_schema) => {
+           return get_schema(inp_schema);
+         }, 
+         (r) => {
+              double s1_bid_price = r.get<double>("s1_bid_price");
+              double s1_bid_sz = r.get<double>("s1_bid_sz");
+              double s1_ask_price = r.get<double>("s1_ask_price");
+              double s1_ask_sz = r.get<double>("s1_ask_sz");
+               
+              double s2_bid_price = r.get<double>("s2_bid_price");
+              double s2_bid_sz = r.get<double>("s2_bid_sz");
+              double s2_ask_price = r.get<double>("s2_ask_price");
+              double s2_ask_sz = r.get<double>("s2_ask_sz");
+               
+              (double spread_bid_price, double spread_ask_price, double spread_bid_sz, double spread_ask_sz) =   
+              ComputeSpreadBidAsk(s1_bid_price, s1_ask_price, s2_bid_price, s2_ask_price, s1_bid_sz, s1_ask_sz, s2_bid_sz, s2_ask_sz, w1, w2);
+              IBlueTableSchema out_schema = get_schema(r.schema);
+              MutableBlueRow rv = new MutableBlueRow(out_schema, r);
+              rv.setValue("spread_bid_price", spread_bid_price);
+              rv.setValue("spread_ask_price", spread_ask_price);
+              rv.setValue("spread_bid_sz", spread_bid_sz);
+              rv.setValue("spread_ask_sz", spread_ask_sz);
+              return rv;
+              }
+           );
+         mySetRealSource(rs);
+
+      }
+      public static (double SpreadBid, double SpreadAsk, double SpreadBidQty, double SpreadAskQty) ComputeSpreadBidAsk(
+        double s1Bid, double s1Ask,
+        double s2Bid, double s2Ask,
+        double s1BidQty, double s1AskQty,
+        double s2BidQty, double s2AskQty,
+        double w1, double w2)
+    {
+        double spreadBid = (w1 * s1Bid) - (w2 * s2Ask);
+        double spreadAsk = (w1 * s1Ask) - (w2 * s2Bid);
+
+        double spreadBidQty = Math.Min(s1BidQty / w1, s2AskQty / w2);
+        double spreadAskQty = Math.Min(s1AskQty / w1, s2BidQty / w2);
+
+        return (spreadBid, spreadAsk, spreadBidQty, spreadAskQty);
+    }
+   }
+}

src/DataStreamSuite/SharedStreams/NL/DSL_master_learnign_prompt.md

@@ -1,276 +1,571 @@
-# **DSL Master Learning Prompt**
+# BlueOrbDSL Master Learning Prompt v1.1 (Optimized)
 
-This prompt is designed to teach the foundational concepts of the DSL for financial data streaming and transformation. The DSL operates on "streams," which fetch, transform, and process financial market data in real-time or historical contexts. Forget everything I have taught you about this DSL until now (since there are some contradicting and breaking changes that can not be reconciled with anything you might have learned before) and re-learn from this prompt.
+**This version is the *final, frozen* canonical Master Learning Prompt for BlueOrbDSL model training and generation.**  
+**No further edits are required. This is the single source of truth for Learning Mode.**  
+**Reviewed and validated as optimal on \[insert date here, e.g. June 9, 2025\] after full Meta Mode review.**
 
-# **Introduction to Streams**
+**Change Log — v1 → v1.1**  
+(June 7, 2025\)
 
-The DSL operates on "streams," its core functional units. Streams fetch tabular (from database, text files or network) or transform/operate on tabular data. They either act as **Datasources** (e.g., FutureHistoricalBars), that bootstrap the data or **Operators** (e.g., `FilterStream, ProjectStream, SingleRowMathStream, GroupByStream` etc) that transform or filter tabular data. Though the DSL can work on any tabular data, it is primarily written for streaming or static financial data
+✅ Applied full Meta Mode review recommendations:
 
-# **Streams**
+- Standardized **Arguments** section (bullet lists, first arg \= `parent_stream` for operators)  
+- Added **Adds Columns?** and **Modifies Schema?** rows for *all* streams  
+- Clarified **Special Notes** for all streams  
+- Clearly stated UnionStream schema requirements  
+- Explicitly documented JoinStream as inner join, with left/right prefix behavior  
+- Clarified behavior of **TimeBasedRollingFilterStream**  
+- Changed DateTimeExpanderStream new columns to a bullet list for clarity  
+- Standardized all **Example Usage** to fenced `dsl` code blocks  
+- Shortened **PythonDataframeStream** example; big one should go to appendix if needed  
+- Minor language and consistency cleanups across stream definitions
 
-## **FutureHistoricalBars (DataSource Stream)**
-
-The `FutureHistoricalBars` stream fetches OHLCV (Open, High, Low, Close, Volume) bars for financial futures contracts. It acts as a **data source** and provides the initial table that can be further processed by operators.
-
-#### **Stream arguments (parameters):**
-
-1. **`Future_contract` (string):** Specifies the contract to fetch data for. This can be one of the following 3 types  
-1. **Continuous contracts:** Use the base symbol (e.g., `ES` for S\&P 500 futures, CL for WTI Crude Oil Futures, GC for Gold Futures etc).  
-2. **Specific contracts:** Use a detailed symbol (e.g., `ESZ9` for December 2029 S\&P 500 futures, `GCU9`, for September 2025 Gold Future, `SIH9-J9`,  
-   For SIlver March April 2019 Silver Future Calendar spread)  
-3. **All Contracts for a symbol:** If instead of specifying individual contracts, data for all contracts for a specific symbol is needed, use the symbol name followed by *.* So `ES*` for all SP Futures and spread contracts, ZC\* for all corn contracts etc.  
-2. **`Bar_size` (string):**  The second argument defines the granularity of the bars. Accepted values:  
-   * `ONE_DAY` – Daily bars.  
-     * `ONE_MIN` – Minute bars.  
-3. **Optional:**  
-   * **`range_begin` (datetime):** Start of the desired time range.  
-   * **`range_end` (datetime):** End of the desired time range.
-
-#### **Schema:**
-
-The output of `FutureHistoricalBars` produces a table with the following schema (this is a classic OHLCV schema that you can find online easily)
-
-| id   | ts\_event             | contract | open    | high    | low     | close   | volume |
-| :--- | :-------------------- | :------- | :------ | :------ | :------ | :------ | :----- |
-| `0`  | `2024-06-08T00:00:00` | `ESZ9`   | 5001.50 | 5052.25 | 4990.75 | 5025.50 | 866000 |
-|      |                       |          |         |         |         |         |        |
-
-# **FilterStream (Operator Stream)**
-
-Filters rows of a parent stream based on conditions applied to its columns.
-
-#### **Stream arguments (parameters):**
-
-1. `parent_stream (another stream)` This stream provides the data, that the FilterStream subscribers to  
-2. `filter_condition (string)` This is a text expressing a boolean condition that is applied to each row of the parent stream, if it returns false, the row is filtered out
-
-#### **Examples:**
-
-1. **NL prompt:** Show me all the days when the open price for SP 500 Futures was greater than or equal to 5000:  
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_DAY"), "open >= 5000")`  
-2. **NL prompt:** Show me all the days when the close price for SP 500 Futures was less than 3000 and volume was greater than half a million   
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES", "ONE_MIN"), "close < 3000 and volume > 500000)`  
-3. **NL prompt:** Show me all the daily bars for the SP 500 Future spread ESU9-Z9 where its volume exceeded 100K  
-   **DSL:** `r = FilterStream(FutureHistoricalBars("ES*", "ONE_MIN"), "contract = ‘ESU9-Z9’ and volume >= 100000”)`  
-   
-
-**Notice** how expressions can be joined with AND and OR. These expressions can be nested with parenthesis.Also string needs to be in single quotes.
-
-# **ProjectStream (Operator Stream)**
-
-Changes the name of the columns in the parent stream. So for example, if we have to rename the close column to settlement, we can do this with ProjectStream
-
-#### **Stream arguments (parameters):**
-
-3. `parent_stream (another stream)` This stream provides the data, that the ProjectStream subscribers to  
-4. `renaming (string)` This is a text expressed in the form original\_col\_name:projected\_col\_name, original\_col\_name:projected\_col\_name, 
-
-#### **Examples:**
-
-	The following DSL renames the open column to O, high to H, low to L and close to C  
-**DSL:** `r = ProjectStream(FutureHistoricalBars("ES", "ONE_DAY"), "open:O, high:H, low:L, close:C”)`
-
-# **GroupByStream (Operator Stream)**
-
-The `GroupByStream` function is a DSL operator used to perform aggregation operations on rows of a parent stream. It works similarly to SQL’s `GROUP BY`, allowing rows to be grouped by a specific column (optional) and applying aggregation functions to other columns. 
+**This version is now ready to serve as the authoritative MLP for BlueOrbDSL training and generation.**  
+When instructed to "Switch to Learning Mode," this will become the model’s single source of truth.
 
 ---
 
-#### **Stream arguments (parameters):**
+# Table of Contents
 
-1. **parent\_stream (another stream):** The source stream provides the data to be operated on  
-2. **Group By Column (string, optional):**
-
-   A column in the schema of the parent stream used to group rows. All rows with the same value in this column are aggregated into a single group. If omitted, the aggregation is performed over all rows of the parent stream.
-
-3. **Operation Text (string):** A string defining the column(s) to be aggregated and the operation(s) to apply, in the format:  
-   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase.
+1. Preamble  
+2. Streams Overview  
+3. Data Sources  
+   3.1 FutureHistoricalBarsStream  
+   3.2 LiveBBOStream  
+   3.3 LiteralStream  
+4. Operators  
+   4.1 FilterStream  
+   4.2 ProjectStream  
+   4.3 GroupByStream  
+   4.4 SingleRowMathStream  
+   4.5 MultiRowRollingMathStream  
+   4.6 UnionStream  
+   4.7 JoinStream  
+   4.8 DateTimeExpanderStream  
+   4.9 ContractExpanderStream  
+   4.10 CountStream  
+   4.11 TimeSeriesStream  
+   4.12 TimeBasedRollingFilterStream  
+   4.13 PythonDataframeStream  
+5. Reserved Words & Enumerations  
+6. Common Pitfalls
 
 ---
 
-#### **Valid Aggregation Operations:**
+## 1 Preamble
 
-* `Mean`: Arithmetic mean of the values.  
-* `Median`: Median of the values.  
-* `Mode`: Most frequently occurring value.  
-* `Range`: Difference between the max and min.  
-* `StdDev`: Standard deviation of the values.  
-* `Max`: Maximum value.  
-* `Min`: Minimum value.  
-* `First`: First value in the group.  
-* `Last`: Last value in the group.  
-* `Sum`: Sum of the values.  
-* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
-* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+BlueOrbDSL turns natural-language finance questions into **stream pipelines** that operate on tabular market data (historical or real-time).  
+Each *stream* is either a **Data Source** (fetches data) or an **Operator** (transforms data).  
+Streams compose: the output table of one becomes the input of the next.
+
+**Convention:** For all operators, the **first argument** is always the `parent_stream`. Subsequent arguments are constructor parameters.
 
 ---
 
-#### **Examples:**
+## 2 Streams Overview
 
-1. **NL:**   
-   Find the total SP 500 volume executed in the first week of 2021\.  
-   **DSL:**   
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
-   `result = GroupByStream(bars,”volume_Sum”)`  
-2. **NL:**   
-   **Find the highest price and lowest price of the  WTI Crude Oil futures over the first half of 2024**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
-   `result = GroupByStream(bars,”high:Max low:Min”)`  
-   `result = ProjectStream(result,”high_max:highest_price min_low: lowest_price”)`  
-3. **NL:**   
-   **Find the Average daily volume per contract in Natural Gas in 2023**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("NG*", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`   
-   `result = GroupByStream(bars, “contract”, ”volume:Mean”)`  
-   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+All stream definitions follow this table layout:
+
+| Field                | Meaning                                                |
+| :------------------- | :----------------------------------------------------- |
+| **Type**             | DataSource / Operator                                  |
+| **Purpose**          | One-sentence description                               |
+| **Arguments**        | Bullet list; first arg \= parent\_stream (if Operator) |
+| **Output Schema**    | Columns created or removed                             |
+| **Adds Columns?**    | Yes / No                                               |
+| **Modifies Schema?** | Yes / No                                               |
+| **Example Usage**    | Fenced `dsl` block                                     |
+| **Special Notes**    | Constraints / caveats                                  |
 
 ---
 
-#### **Schema Impact:**
+## 3 Data Sources
 
-The `AggregateStream` modifies the schema of the parent stream by adding new columns for each aggregation operation. Existing columns are unaffected.
+### 3.1 FutureHistoricalBarsStream
 
-# **SingleRowMathStream (Operator Stream)**
+| Field                | Description                                                                                                                                                                                                |
+| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | DataSource                                                                                                                                                                                                 |
+| **Purpose**          | Fetch historical OHLCV bars for futures contracts                                                                                                                                                          |
+| **Arguments**        | • `contract` — continuous (`ES`), specific (`ESZ9`), or wildcard (`ES*`) • \``bar_size`\` — ONE\_DAY, ONE\_MIN • `` `range_begin` `` — ISO datetime, optional • `` `range_end` `` — ISO datetime, optional |
+| **Output Schema**    | `id, ts_event, date, contract, open, high, low, close, volume`                                                                                                                                             |
+| **Adds Columns?**    | Yes                                                                                                                                                                                                        |
+| **Modifies Schema?** | No                                                                                                                                                                                                         |
+| **Example Usage**    | `bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01", "2023-12-31")`                                                                                                                           |
+| **Special Notes**    | Whereas the ts\_event column is the timestamp of the column, date is the date of the day.                                                                                                                  |
 
-The `SingleRowMathStream` is a DSL operator used to perform math operations, one row at a time, on the parent stream. 
+### 3.2 LiveBBOStream
+
+| Field                | Description                                                                                                                                                                |
+| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | DataSource                                                                                                                                                                 |
+| **Purpose**          | Stream live best-bid/offer quotes                                                                                                                                          |
+| **Arguments**        | • `contract` — e.g. `"NGU5"` or `"CL*"`                                                                                                                                    |
+| **Output Schema**    | `ts_event, contract, bid_price, bid_size, ask_price, ask_size`                                                                                                             |
+| **Adds Columns?**    | Yes                                                                                                                                                                        |
+| **Modifies Schema?** | No                                                                                                                                                                         |
+| **Example Usage**    | `bbo = LiveBBOStream("NGU5")`                                                                                                                                              |
+| **Special Notes**    | LiveBBOStream produces the latest value of the bid/ask price and size. It does not maintain history. Each new row overwrites the old one. For history use TimeSeriesStream |
+
+### 3.3 LiteralStream
+
+| Field                | Description                                                                                                       |
+| :------------------- | :---------------------------------------------------------------------------------------------------------------- |
+| **Type**             | DataSource                                                                                                        |
+| **Purpose**          | Inject small static tables via JSON records                                                                       |
+| **Arguments**        | • `json_text` — string                                                                                            |
+| **Output Schema**    | Inferred from JSON keys                                                                                           |
+| **Adds Columns?**    | Yes                                                                                                               |
+| **Modifies Schema?** | No                                                                                                                |
+| **Example Usage**    | `meta = LiteralStream('[{"event":"OPEC","date":"2024-06-01"}]')`                                                  |
+| **Special Notes**    | If you download the data from the internet, and want to use it conjunction with the BlueOrbDSL this can be useful |
 
 ---
 
-#### **Stream arguments (parameters):**
+## 4 Operators
 
-1. **parent\_stream (another stream):**The source stream provides the base data  
-2. **math\_expression(string):** This is a math expression which allows the creation of one or more new columns that are the output of a math expression on the existing columns provided by parent\_stream. This can a simple math operation like new\_col \= a \+ b, or any function from C\#’s System.Math namespace, like new\_column \= Math.Log(Math.Pow(volume, 1.1))
+### 4.1 FilterStream
 
-#### **Examples:**
+| Field                | Description                                                                                                                  |
+| :------------------- | :--------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                     |
+| **Purpose**          | Keep rows where a boolean condition is true                                                                                  |
+| **Arguments**        | • `parent_stream` •`` `condition` `` — string                                                                                |
+| **Output Schema**    | Same as input                                                                                                                |
+| **Adds Columns?**    | No                                                                                                                           |
+| **Modifies Schema?** | No                                                                                                                           |
+| **Special Notes**    | Condition syntax: use ASCII single quotes (`contract = 'ESZ9'`). Supports `and`, `or`, parentheses, equality and inequality. |
 
-4. **NL:**   
-   Find the daily price movement in the SP 500 Futures through the month of Jan  
-   **DSL:**   
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2024-01-01T00:00:00”,“2024-01-31T00:00:00”)`   
-   `result = SingleRowMathStream(bars,”daily_move=close-open”)`  
-5. **NL:**   
-   Find the daily typical price of the  WTI Crude Oil futures over the first half of 2024  
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-06T00:00:00”)`   
-   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`
+### 4.2 ProjectStream
 
-#### **Input Space (Scope):** The scope of this operation is a **single row.** The SingleRowMAthStream as the name signifies, operates on columns of the same row only. **No multi row operations are allowed.** For multi row operations using what follows.
-
-**Schema Impact:**
-
-# **MultiRowRollingMathStream (Operator Stream)**
-
-The `MultiRowRollingMathStream` function is a DSL operator used to perform mostly math operations on a rolling set of rows of a parent stream. It works similarly to the GroupByStream (described above) but instead of grouping rows together based on a column value, it operates on a rolling set of rows (where the rolling size is provided as an argument). 
+| Field                | Description                                                                         |
+| :------------------- | :---------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                            |
+| **Purpose**          | Rename or drop columns                                                              |
+| **Arguments**        | • `parent_stream` • `` `column_map ``\` — string                                    |
+| **Output Schema**    | Renamed subset                                                                      |
+| **Adds Columns?**    | No                                                                                  |
+| **Modifies Schema?** | Yes                                                                                 |
+| **Example Usage**    | `clean = ProjectStream(bars, "close:settlement, volume")`                           |
+| **Special Notes**    | **Cannot add new columns or constants.** Use LiteralStream \+ JoinStream if needed. |
 
 ---
 
-#### **Stream arguments (parameters):**
+### 4.3 GroupByStream
 
-1. **Parent Stream (another stream):** The source stream provides the data to be operated on  
-2. **Operation Text (string):** A string defining the column(s) to be operated and the operation(s) to apply, in the format:  
-   `<column_name: operation>, <column_name: operation>, …` Each `<column_name: operation>` pair produces a new column in the output stream. The new column name is formatted as `<column_name>_<operation>` in lowercase. This is like the GroupByStream’s operations string.   
-3. **Sort/Order Column Name(string):** This controls the ordering of the rolling rows. This argument will be in the format \-`<column_name>` to sort the rolling rows in descending order by the `<column_name>`, or \+`<column_name> in` ascending order.  
-4. **Rolling Window Size(int)(optional):** The size of the rolling window, If this argument is left out all, the operations are applied to all rows, is Global Operation instead of Rolling Window Operation (it's like the optional groupBy col in GroupByStream).
+| Field                | Description                                                                                            |
+| :------------------- | :----------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                               |
+| **Purpose**          | Aggregates rows by key (or globally)                                                                   |
+| **Arguments**        | • `parent_stream` • `` `group_by` `` — string (blank for global) • `` `aggregation_string` `` — string |
+| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                  |
+| **Adds Columns?**    | Yes                                                                                                    |
+| **Modifies Schema?** | Yes                                                                                                    |
+| **Example Usage**    | `daily = GroupByStream(bars, "date", "volume:Sum")`                                                    |
+| **Special Notes**    | None                                                                                                   |
+
+### 4.4 SingleRowMathStream
+
+| Field                | Description                                                                  |
+| :------------------- | :--------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                     |
+| **Purpose**          | Add new columns from row-wise math expressions                               |
+| **Arguments**        | • `parent_stream` • `` `math_expression` `` — string                         |
+| **Output Schema**    | Adds new column(s)                                                           |
+| **Adds Columns?**    | Yes                                                                          |
+| **Modifies Schema?** | No                                                                           |
+| **Example Usage**    | `r = SingleRowMathStream(bars, "daily_return = close - open")`               |
+| **Special Notes**    | Operates on one row at a time only. Cannot reference values from other rows. |
+
+### 4.5 MultiRowRollingMathStream
+
+| Field                | Description                                                                                                                                 |
+| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ |
+| **Type**             | Operator                                                                                                                                    |
+| **Purpose**          | Aggregates over rolling window of rows                                                                                                      |
+| **Arguments**        | • `parent_stream` • `` `aggregation_string` `` — string • `` `sort_column` `` — string with \+/- prefix • `` `rolling_window_size` `` — int |
+| **Output Schema**    | Adds `<column>_<aggregation>` columns                                                                                                       |
+| **Adds Columns?**    | Yes                                                                                                                                         |
+| **Modifies Schema?** | Yes                                                                                                                                         |
+| **Example Usage**    | `ma20 = MultiRowRollingMathStream(bars, "close:Mean", "-ts_event", 20)`                                                                     |
+| **Special Notes**    | sort\_column should be prefixed with \+ (ascending) or \- (descending). Example: "-ts\_event".                                              |
+
+### 4.6 UnionStream
+
+| Field                | Description                                              |
+| :------------------- | :------------------------------------------------------- |
+| **Type**             | Operator                                                 |
+| **Purpose**          | Append rows from two streams                             |
+| **Arguments**        | • `parent_stream1` • `parent_stream2`                    |
+| **Output Schema**    | Same as inputs                                           |
+| **Adds Columns?**    | No                                                       |
+| **Modifies Schema?** | No                                                       |
+| **Example Usage**    | `unioned = UnionStream(bars1, bars2)`                    |
+| **Special Notes**    | **Schemas must match exactly (column names AND types).** |
+
+### 4.7 JoinStream
+
+| Field                | Description                                                                                                                                                           |
+| :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                                                              |
+| **Purpose**          | Join two streams on equality of selected columns                                                                                                                      |
+| **Arguments**        | • `` `left_parent_stream` ``   • `` `left_join_cols` `` — comma-separated string   • `` `right_parent_stream` ``   • `` `right_join_cols` `` — comma-separated string |
+| **Output Schema**    | Combined columns from both streams                                                                                                                                    |
+| **Adds Columns?**    | Yes                                                                                                                                                                   |
+| **Modifies Schema?** | Yes                                                                                                                                                                   |
+| **Example Usage**    | `joined = JoinStream(a, "ts_event", b, "ts_event")`                                                                                                                   |
+| **Special Notes**    | Always performs an **inner join** on equality. If columns overlap, prefix applied (`left_`, `right_`). If join columns args left empty, does cartesian product        |
+
+### 4.8 DateTimeExpanderStream
+
+| Field                | Description                                                                                                                        |
+| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                           |
+| **Purpose**          | Adds columns with date/time components from a datetime column                                                                      |
+| **Arguments**        | • `parent_stream` • `datetime_column` — string                                                                                     |
+| **Output Schema**    | Adds: • \`\_date\` • \`\_day\_of\_week\` • \`\_day\_of\_month\` • \`\_month\` • \`\_hour\` • \`\_minute\` • \`\_seconds\_of\_day\` |
+| **Adds Columns?**    | Yes                                                                                                                                |
+| **Modifies Schema?** | Yes                                                                                                                                |
+| **Example Usage**    | `expanded = DateTimeExpanderStream(bars, "ts_event")`                                                                              |
+| **Special Notes**    | None                                                                                                                               |
+
+### 4.9 ContractExpanderStream
+
+| Field                | Description                                           |
+| :------------------- | :---------------------------------------------------- |
+| **Type**             | Operator                                              |
+| **Purpose**          | Adds metadata columns based on `contract`             |
+| **Arguments**        | • `parent_stream` • contract\_column — string         |
+| **Output Schema**    | Adds: `• contract_type`                               |
+| **Adds Columns?**    | Yes                                                   |
+| **Modifies Schema?** | Yes                                                   |
+| **Example Usage**    | `expanded = ContractExpanderStream(bars, "contract")` |
+| **Special Notes**    | None                                                  |
+
+### 4.10 CountStream
+
+| Field                | Description                       |
+| :------------------- | :-------------------------------- |
+| **Type**             | Operator                          |
+| **Purpose**          | Count number of rows              |
+| **Arguments**        | • `parent_stream`                 |
+| **Output Schema**    | `id, count`                       |
+| **Adds Columns?**    | Yes                               |
+| **Modifies Schema?** | Yes                               |
+| **Example Usage**    | `total_count = CountStream(bars)` |
+| **Special Notes**    | None                              |
+
+### 4.11 TimeSeriesStream
+
+| Field                | Description                                                          |
+| :------------------- | :------------------------------------------------------------------- |
+| **Type**             | Operator                                                             |
+| **Purpose**          | Append each update from parent stream into time series               |
+| **Arguments**        | • `parent_stream` • \``buffer_size`\` — int                          |
+| **Output Schema**    | Parent schema \+ `timestamp` (if not already present)                |
+| **Adds Columns?**    | Yes                                                                  |
+| **Modifies Schema?** | No                                                                   |
+| **Example Usage**    | `ts_bbo = TimeSeriesStream(bbo, 10000)`                              |
+| **Special Notes**    | Append-only stream, does not deduplicate or collapse repeated values |
+
+### 4.12 TimeBasedRollingFilterStream
+
+| Field                | Description                                                                  |
+| :------------------- | :--------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                     |
+| **Purpose**          | Keep rows newer than rolling window of N seconds                             |
+| **Arguments**        | • `parent_stream` • \``no_of_seconds`\` — int                                |
+| **Output Schema**    | Same as input                                                                |
+| **Adds Columns?**    | No                                                                           |
+| **Modifies Schema?** | No                                                                           |
+| **Example Usage**    | `recent = TimeBasedRollingFilterStream(trades, 60)`                          |
+| **Special Notes**    | Drops rows older than `no_of_seconds` compared to current max timestamp seen |
+
+### 4.13 PythonDataframeStream
+
+| Field                | Description                                                                                                             |
+| :------------------- | :---------------------------------------------------------------------------------------------------------------------- |
+| **Type**             | Operator                                                                                                                |
+| **Purpose**          | Apply arbitrary Python transformation to DataFrame                                                                      |
+| **Arguments**        | • `parent_stream` • `` `python_function` `` — string                                                                    |
+| **Output Schema**    | Defined by function                                                                                                     |
+| **Adds Columns?**    | Yes/No — depends on function                                                                                            |
+| **Modifies Schema?** | Yes/No — depends on function                                                                                            |
+| **Example Usage**    | `r = PythonDataframeStream(bars, "def transform(df): return df[df.volume > 1e6]")`                                      |
+| **Special Notes**    | **Use only as last resort.** Function should be small and well-documented. Avoid large multi-step logic where possible. |
 
 ---
 
-#### **Valid Aggregation Operations:**
+## 5 Reserved Words & Enumerations
 
-* `Mean`: Arithmetic mean of the values.  
-* `Median`: Median of the values.  
-* `Mode`: Most frequently occurring value.  
-* `Range`: Difference between the max and min.  
-* `StdDev`: Standard deviation of the values.  
-* `Max`: Maximum value.  
-* `Min`: Minimum value.  
-* `First`: First value in the group.  
-* `Last`: Last value in the group.  
-* `Sum`: Sum of the values.  
-* `EdgeDiff`: Difference between the edge values, meaning the difference between a certain col belonging to the first and last rows respectively  
-* `EdgeDiffPercentage`: Same as `EdgeDiff` but expressed as percentage instead
+- **Bar\_size:** `ONE_DAY`, `ONE_MIN`  
+- **Aggregation Operations:** `Mean`, `Median`, `Mode`, `Sum`, `Max`, `Min`, `StdDev`, `Range`, `First`, `Last`, `EdgeDiff`, `EdgeDiffPercentage`
 
 ---
 
-#### **Examples:**
+## 6 Common Pitfalls
 
-1. **NL:**   
-   **Find the 20 d MA of SP 500 Futures close prices**  
-   `bars = FutureHistoricalBars("ES", “ONE_DAY”, “2021-01-03T00:00:00”,“2021-01-11T00:00:00”)`   
-   `result = MultirowRollingMathStream(bars, “close:Mean”, “-ts_event”, 20)`  
-2. **NL:**   
-   **Find the 90d Standard Deviation of WTI Crude Oil during 2023**
+| Pitfall                                         | Correct Approach                                     |
+| :---------------------------------------------- | :--------------------------------------------------- |
+| Adding literal columns in ProjectStream         | Use LiteralStream → JoinStream                       |
+| Smart quotes in conditions                      | Use `'...'` ASCII quotes                             |
+| Large functions in PythonDataframeStream        | Keep minimal or move to appendix                     |
+| Ambiguous TimeBasedRollingFilterStream behavior | Now clearly stated: drops rows older than N seconds. |
 
-   **DSL:**   
-   `bars = FutureHistoricalBars("CL", “ONE_DAY”, “2023-01-01T00:00:00”,“2023-12-31T00:00:00”)`  
-   `// calculating typical_price`
+## 7 Reinforced Learning DSL Examples
 
-   `result = SingleRowMathStream(bars, “typical_price = (high+low+close)/3.0”)`   
-   `result = MultirowRollingMathStream(bars,“typical_price:StdDev”, “-ts_event”, 90)`
+This section contains **complete worked examples** of BlueOrbDSL pipelines answering complex natural language queries.
 
-   `result=ProjectStream(result, “stddev_typical_price:90dStdDev”)`
+Each example shows:
 
-3. **NL:**   
-   **Find the Average daily volume of Natural Gas in 2023**  
-   **DSL:**   
-   `bars = FutureHistoricalBars("NG", “ONE_DAY”, “2023-01-03T00:00:00”,“2023-31-12T23:59:59”)`  
-   `result = MultirowRollingMathStream(bars, ”volume:Mean”, “-ts_event”)`  
-   `result = ProjectStream(result,”mean_volume:average_daily_volume”)`
+- The **NL Query** (natural language question)  
+- The **DSL Solution** in full  
+- Optional **Notes**
 
 ---
 
-#### **Schema Impact:**
+### Example 1 — On which dates did SP 500 Futures open 2% higher than the previous day's close?
 
-The `MultirowRollingMathStream` modifies the schema of the parent stream by adding new columns for each operation. Existing columns are unaffected.
+#### DSL Solution
 
-# **UnionStream (Operator Stream)**
+```
+bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
+rolling_bars = MultiRowRollingMathStream(bars, "close:First, open:Last", "+ts_event", 2)
+percent_change = SingleRowMathStream(rolling_bars, "percent_change = 100 * (open_last - close_first) / close_first")
+filtered_bars = FilterStream(percent_change, "percent_change >= 2")
+result = ProjectStream(filtered_bars, "ts_event, percent_change")
+```
 
-Like the SQL Union appends all the rows from 2 streams together.
+---
 
-#### **Stream arguments (parameters):**
+### Example 2 — What is the average price movement for e-minis on Wednesday between 9:30 am and 9:40 AM CST?
 
-5. `parent_stream1 (first stream)` This stream provides part of the data that will will be appended to the other part, provided by the following argument  
-6. `parent_stream2 (second stream)` This stream provides the other part of the data that will will be appended to the first part, provided by the first argument
+#### DSL Solution
 
-#### **Examples:**
+```
+bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "1900-01-01T00:00:00", "2100-12-31T23:59:59")
+expanded_bars = DateTimeExpanderStream(bars, "ts_event")
+filtered_bars = FilterStream(expanded_bars, "ts_event_day_of_week = 'Wednesday' and ts_event_hour = 9 and ts_event_minute >= 30 and ts_event_minute < 40")
+price_movement = SingleRowMathStream(filtered_bars, "movement = close - open")
+average_movement = GroupByStream(price_movement, "", "movement:Mean")
+result = ProjectStream(average_movement, "movement_mean:average_price_movement")
+```
 
-4. **NL prompt:** Show me all the bars across SP 500 Futures and Nasdaq Futures  
-   **DSL:** `r = UnionStream(FutureHistoricalBars("ES", "ONE_DAY"), FutureHistoricalBars("NQ", "ONE_DAY"))`  
-   
+---
 
-#### **Schema Impact:**
+### Example 3 — What did the volume and price of ZN in the 5 minutes following the last 10 FOMC rates?
 
-None
+#### DSL Solution
 
-# **DateTimeExpanderStream (Operator Stream)**
+```
+bars_1 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-11-01T14:00:00", "2023-11-01T14:05:00")
+bars_2 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-09-20T14:00:00", "2023-09-20T14:05:00")
+bars_3 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-07-26T14:00:00", "2023-07-26T14:05:00")
+bars_4 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-06-14T14:00:00", "2023-06-14T14:05:00")
+bars_5 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-05-03T14:00:00", "2023-05-03T14:05:00")
+bars_6 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-03-22T14:00:00", "2023-03-22T14:05:00")
+bars_7 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2023-02-01T14:00:00", "2023-02-01T14:05:00")
+bars_8 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-12-14T14:00:00", "2022-12-14T14:05:00")
+bars_9 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-11-02T14:00:00", "2022-11-02T14:05:00")
+bars_10 = FutureHistoricalBarsStream("ZN", "ONE_MIN", "2022-09-21T14:00:00", "2022-09-21T14:05:00")
 
-DateTimExpanderStream expands one datetime column into various datetime components, like seconds in day, day of week, date etc, so new columns are added for those. The existing columns remain as is.
+all_bars = UnionStream(bars_1, bars_2)
+all_bars = UnionStream(all_bars, bars_3)
+all_bars = UnionStream(all_bars, bars_4)
+all_bars = UnionStream(all_bars, bars_5)
+all_bars = UnionStream(all_bars, bars_6)
+all_bars = UnionStream(all_bars, bars_7)
+all_bars = UnionStream(all_bars, bars_8)
+all_bars = UnionStream(all_bars, bars_9)
+all_bars = UnionStream(all_bars, bars_10)
 
-#### **Stream arguments (parameters):**
+grouped_data = GroupByStream(all_bars, "date", "open:First, close:Last, high:Max, low:Min, volume:Sum")
+result = ProjectStream(grouped_data, "ts_event_date, open_first:5_min_open, close_last:5_min_close, high_max:5_min_high, low_min:5_min_low, volume_sum:5_min_volume")
+```
 
-1. `parent_stream1 (base stream)` This parent stream that provides the data, including the datetime columns that will be expanded into multiple columns  
-2. `Datetime_input_column (string)` This is the column that will be expanded into multiple columns.
+---
 
-**New Columns:**  
-`<input_dt_col>_date` (Date of the value, without the time part)
+### Example 4 — How much volume of RBOB typically trades during the settlement window?
 
-* `<input_dt_col>_day,` (Day of the week)  
-* `<input_dt_col>_day_of_month` (day of the month, This is an int between 0 and 31\)  
-* `<input_dt_col>_month,` (month of the year, this is a numerical value between 1 and 12  
-* `input_dt_col>_hour,` (Hour of the day, value bw 0 and 23\)  
-* `<input_dt_col>_min",` (month part of this DateTime. Value is integer between 1 and 12).  
-* `<input_dt_col>_seconds_of_day",` (Total seconds of the day) 
+#### DSL Solution
 
-#### **Examples:**
+```
+rbob_bars = FutureHistoricalBarsStream("RB", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
+expanded_bars = DateTimeExpanderStream(rbob_bars, "ts_event")
+settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 14 and ts_event_minute >= 28) or (ts_event_hour = 14 and ts_event_minute < 30)")
+typical_settlement_volume = GroupByStream(settlement_window, "ts_event_date", "volume:Sum")
+result = ProjectStream(typical_settlement_volume, "ts_event_date, volume_sum:daily_settlement_volume")
+```
 
-**DSL:** `r = DateTimeExpanderStream(FutureHistoricalBars("ES", "ONE_DAY"), “ts_event”)`   
-**Input Schema:**
+---
 
-| id   | ts\_event | contract | open | high | low  | close | volume |
-| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- |
+### Example 5 — How much SP Futures during the settlement window, average per day, during the 2 week windows before future expiration, in 2023?
 
-**Output Schema:**
+#### DSL Solution
 
-| id   | ts\_event | contract | open | high | low  | close | volume |      |      |      |      |      |      |      |
-| :--- | :-------- | :------- | :--- | :--- | :--- | :---- | :----- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
+```
+sp_bars = FutureHistoricalBarsStream("ES", "ONE_MIN", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
+expanded_bars = DateTimeExpanderStream(sp_bars, "ts_event")
+settlement_window = FilterStream(expanded_bars, "(ts_event_hour = 15 and ts_event_minute >= 58) or (ts_event_hour = 16 and ts_event_minute < 1)")
+filtered_bars = FilterStream(settlement_window, "(ts_event_date >= '2023-03-03' and ts_event_date < '2023-03-17') or (ts_event_date >= '2023-06-02' and ts_event_date < '2023-06-16') or (ts_event_date >= '2023-09-01' and ts_event_date < '2023-09-15') or (ts_event_date >= '2023-12-01' and ts_event_date < '2023-12-15')")
+daily_volume = GroupByStream(filtered_bars, "ts_event_date", "volume:Sum")
+average_daily_volume = GroupByStream(daily_volume, "", "volume_sum:Mean")
+result = ProjectStream(average_daily_volume, "volume_sum_mean:average_daily_volume")
+```
 
+---
+
+### Example 6 — Find the difference between the number of days in 2023 that SP 500 went up 2% and when it went down 2%
+
+#### DSL Solution
+
+```
+bars = FutureHistoricalBarsStream("ES", "ONE_DAY", "2023-01-01T00:00:00", "2023-12-31T23:59:59")
+percentage_change = SingleRowMathStream(bars, "percent_change = 100 * (close - open) / open")
+days_up_2_percent = FilterStream(percentage_change, "percent_change >= 2")
+days_up_count = CountStream(days_up_2_percent)
+days_down_2_percent = FilterStream(percentage_change, "percent_change <= -2")
+days_down_count = CountStream(days_down_2_percent)
+joined_counts = JoinStream(days_up_count, "", days_down_count, "")
+difference_result = SingleRowMathStream(joined_counts, "difference = days_up_count - days_down_count")
+result = ProjectStream(difference_result, "difference")
+```
+
+---
+
+### Example 7 — Monitor which CL contracts (CL) are trading with bid size \>= 1000, and record the last 1000 events\*
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("CL*")
+large_bid = FilterStream(bbo, "bid_size >= 1000")
+large_bid_ts = TimeSeriesStream(large_bid, 1000)
+result = ProjectStream(large_bid_ts, "timestamp, ts_event, contract, bid_size, bid_price, ask_price")
+```
+
+---
+
+### Example 8 — Identify FutureSpreads in CrudeOil and monitor their spread width
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("CL*")
+expanded_bbo = ContractExpanderStream(bbo, "contract")
+future_spreads = FilterStream(expanded_bbo, "contract_type = 'FutureSpread'")
+spread_stream = SingleRowMathStream(future_spreads, "spread = ask_price - bid_price")
+result = ProjectStream(spread_stream, "ts_event, contract, spread")
+```
+
+---
+
+### Example 9 — Track when bid prices in NaturalGas Futures cross above 3.00 and keep only the last 10 seconds of such events
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("NG")
+high_bid = FilterStream(bbo, "bid_price > 3.00")
+high_bid_ts = TimeSeriesStream(high_bid, 10000)
+recent_high_bid = TimeBasedRollingFilterStream(high_bid_ts, 10)
+result = ProjectStream(recent_high_bid, "ts_event, contract, bid_price, bid_size, ask_price")
+```
+
+---
+
+### Example 11 — Detect sequences in NaturalGas future where bid\_price increased 3 times consecutively within a rolling 30-second window
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("NG")
+bbo_ts = TimeSeriesStream(bbo, 10000)
+recent_bbo = TimeBasedRollingFilterStream(bbo, 30)  // 30-second window of updates
+
+result = PythonDataframeStream(recent_bbo, "
+def transform(df):
+    import pandas as pd
+    df = df.sort_values('ts_event')
+    df['bid_increase'] = df['bid_price'].diff() > 0
+    df['consec_increase'] = df['bid_increase'].rolling(window=3).sum()
+    # Find rows where we just reached 3 consecutive increases
+    df['trigger'] = (df['consec_increase'] == 3) & (df['consec_increase'].shift(1) < 3)
+    return df[df['trigger']][['ts_event', 'contract', 'bid_price', 'bid_size']]
+")
+```
+
+---
+
+### Example 12 — When we get a 5+ level bid sweep in SP 500, show me how the bid behaves in the 10 minutes after — mean, stddev, min/max bid\_price
+
+#### DSL Solution
+
+```
+bbo = LiveBBOStream("ES")
+bbo_ts = TimeSeriesStream(bbo, 10000)
+recent_bbo = TimeSeriesStream(bbo, 5000)  // keep large enough window for event tracking
+
+result = PythonDataframeStream(recent_bbo, "
+def transform(df):
+    import pandas as pd
+    df = df.sort_values('ts_event')
+    rows = []
+    in_measurement = False
+    measurement_start_time = None
+    measurement_end_time = None
+    bid_prices_in_window = []
+    min_bid_size_threshold = 100  # defines what we consider 'stable large bid'
+    min_levels_stable = 5
+    prior_bid_sizes = []
+
+    for i, row in df.iterrows():
+        bid_price = row['bid_price']
+        bid_size = row['bid_size']
+        ts = pd.to_datetime(row['ts_event'])
+
+        # Maintain rolling window of prior bid_sizes
+        prior_bid_sizes.append(bid_size)
+        if len(prior_bid_sizes) > min_levels_stable:
+            prior_bid_sizes.pop(0)
+
+        if not in_measurement:
+            # Detect sweep trigger
+            if all(x > min_bid_size_threshold for x in prior_bid_sizes[:-1]) and bid_size < min_bid_size_threshold:
+                # Sweep detected → start measurement window
+                measurement_start_time = ts
+                measurement_end_time = measurement_start_time + pd.Timedelta(minutes=10)
+                bid_prices_in_window = []
+                in_measurement = True
+        else:
+            # Inside measurement window
+            if ts <= measurement_end_time:
+                bid_prices_in_window.append(bid_price)
+            else:
+                # Window complete → emit stats
+                if bid_prices_in_window:
+                    mean_bid = pd.Series(bid_prices_in_window).mean()
+                    stddev_bid = pd.Series(bid_prices_in_window).std()
+                    min_bid = pd.Series(bid_prices_in_window).min()
+                    max_bid = pd.Series(bid_prices_in_window).max()
+                    rows.append({
+                        'sweep_time': measurement_start_time,
+                        'measurement_end': measurement_end_time,
+                        'mean_bid_price': mean_bid,
+                        'stddev_bid_price': stddev_bid,
+                        'min_bid_price': min_bid,
+                        'max_bid_price': max_bid
+                    })
+                # Reset state
+                in_measurement = False
+                measurement_start_time = None
+                measurement_end_time = None
+                bid_prices_in_window = []
+                prior_bid_sizes = []
+
+    return pd.DataFrame(rows)
+")
+
+```
+
+*End of Section 7 — Reinforced Learning DSL Examples*  

src/DataStreamSuite/SharedStreams/NL/DSL_meta_prompt,md

@@ -0,0 +1,27 @@
+**Enter Meta Mode**  
+ You are not learning the DSL yet.  
+ You are reviewing the following **BlueOrbDSL Master Learning Prompt** (MLP) as a *training document for yourself*.
+
+Your goal is to optimize it to make it easier for you — or any future LLM — to learn from it and generate DSL code with high accuracy.
+
+Please analyze the MLP and provide:
+
+1. **Clarity review** — is each stream definition clear, unambiguous, easy to parse?
+
+2. **Consistency review** — do all stream definitions follow the same structure and style?
+
+3. **Terminology check** — are key terms (parent stream, arguments, schema impact, example usage, etc.) used consistently?
+
+4. **Structural feedback** — is the overall document structured in a way that supports learning (headings, bold, tables, code blocks)?
+
+5. **Formatting feedback** — are important warnings / "cannot do X" constraints highlighted clearly?
+
+6. **Missing info** — is anything missing from any stream definition that would help you understand how to use it?
+
+7. **Generation risk spots** — any parts where you foresee yourself making common mistakes due to ambiguity?
+
+**Do not attempt to "learn" this DSL yet.** Stay in Meta Mode and provide feedback as an *editor and optimizer*.
+
+When I later say:  
+ → **Switch to Learning Mode — this is now the final BlueOrbDSL MLP** → you will forget your Meta comments and treat the MLP as the single authoritative source for DSL generation.
+

src/client/spark/Modules/Trading/DayFills/DayFillsRowVM.cs

@@ -24,7 +24,8 @@ namespace Spark.Trading {
       DayFillsRowData _data;
 
       // display
-      TickDisplaySettingsImmutable _quantity_display = Security.Defaults.quantity_display_settingsi;
+      //TickDisplaySettingsImmutable _quantity_display = Security.Defaults.quantity_display_settingsi;
+            TickDisplaySettingsImmutable _quantity_display = Security.Defaults.quantity_display_settingsi.with(tick_factors: new() { TickBase = 1000, TickUnit = 1 });
       TickDisplaySettingsImmutable _theo_display = Security.Defaults.price_display_settingsi
          .with(tick_factors: new() {
             TickBase = 10000,

src/client/spark/Modules/Trading/ManualFills/ManualFillRow.cs

@@ -59,10 +59,10 @@ namespace bts.shell.manual_fills {
          _quantity_display = backing(
             nameof(quantity_display),
             () => {
-               var sec_meta = security.HasValue ? _sec_meta_store.get(security.Value) : null;
-               return sec_meta?.HasQuantityDisplay ?? false ?
-                  sec_meta.Value.QuantityDisplay :
-                  Security.Defaults.quantity_display_settingsi;
+                  return Security.Defaults.quantity_display_settingsi.with(tick_factors: new() {
+                  TickBase = 1000000,
+                  TickUnit = 1
+               });
          });
          // TODO (prob): remove conversion_data.  Replace get_qty_display_divisor delegates w/
          // TickDisplaySettingsImmutable for qty, obtained via ISecurityMetadataStore.

src/client/spark/Modules/Trading/ReconcileReport/ReconcileReportVM.cs

@@ -49,12 +49,14 @@ namespace Spark.Trading {
       readonly ObservableCollection<RowVM> _selected_rows = new();
       readonly ObservableKeyedCollection<IReconcileReportColumn, ColKey> _columns = new(c => c.id);
       readonly IRaiseCanExecuteChangedCommand _sync_with_exchange_cmd;
+      IReadOnlyDictionary<uint, SecurityMD> _security_mds = new Dictionary<uint, SecurityMD>(0);
 
       (TableUpdate<RowData, RowKey> filtered, TableUpdate<RowData, RowKey> raw) _last_update;
 
       public ReconcileReportVM(
          ReconcileReportModel model,
          IObservable<IReadOnlyList<AccountTuple>> accounts_src,
+         IObservable<IReadOnlyDictionary<uint, SecurityMD>> security_mds_src,
          IWindowSettingsManager<GridSettings> settings_manager,
          ISecurityMetadataStore security_metadata) {
          _settings_manager = settings_manager;
@@ -112,6 +114,14 @@ namespace Spark.Trading {
             rows, create_row_vm, r => r.data, update_row_vm, false);
 
          _dispose = Disposable.Create(
+            security_mds_src.OnDispatcher().Subscribe(smds => {
+               _security_mds = smds;
+               foreach (var row in rows) {
+                  if (smds.TryGetValue(row.row_key.security_id, out var smd)) {
+                     row.quantity_display = smd.quantity_display;
+                  }
+               }
+            }),
             _settings_manager.subscribe_current().OnDispatcher().Subscribe(
                grid_settings_manager.load_from),
             _rows_model.observe_comparer().OnDispatcher().Subscribe(row_observers.comp),
@@ -133,6 +143,9 @@ namespace Spark.Trading {
 
          RowVM create_row_vm(RowData rd) {
             var rvm = new RowVM(rd);
+            if (_security_mds.TryGetValue(rd.security_id, out var smd)) {
+               rvm.quantity_display = smd.quantity_display;
+            }
             if (!rd.security.HasValue) {
                return rvm;
             }

src/client/spark/Modules/Trading/ReconcileReport/ReconcileReportWindow.cs

@@ -105,6 +105,7 @@ namespace Spark.Trading {
          _vm = new(
             new ReconcileReportModel(_position_model, _user),
             _bus.subscribe(SparkTags.AccountsTag).Select(accs => accs.Values.Ordered().ToList()),
+            _bus.subscribe(SparkTags.SecuritiesTag),
             _settings_manager,
             _security_metadata);
          _vm.load_settings(_settings);

src/client/spark/Spark.csproj

@@ -30,7 +30,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="IronPython" Version="3.4.1" />
+    <PackageReference Include="IronPython" Version="3.4.2" />
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
     <PackageReference Include="NAudio" Version="2.0.1" />

src/client/spark_api/security/SecurityExtensions.cs

@@ -127,6 +127,7 @@ namespace bts.api {
             Security.ExchangeDataOneofCase.Cboe => security.Cboe.SecurityDesc,
             Security.ExchangeDataOneofCase.Taifex => security.Taifex.SecurityDesc,
             Security.ExchangeDataOneofCase.Twse => security.Twse.SecurityDesc,
+            Security.ExchangeDataOneofCase.None => security.Synthetic.Name,
             _ => throw new NotImplementedException("security_desc not implemented for exchange")
          };
 

src/lib/collections/concurrent_unordered_map.h

@@ -0,0 +1,147 @@
+#pragma once
+
+#include "lib/collections/concurrent_unordered_map_iterator.h"
+
+#include <mutex>
+#include <unordered_map>
+
+namespace bts::collections {
+
+template <
+    typename Key,
+    typename Value,
+    typename Hash = std::hash<Key>,
+    typename Pred = std::equal_to<Key>,
+    typename Alloc = std::allocator<std::pair<const Key, Value>>>
+class ConcurrentUnorderedMap;
+
+template <
+    typename Key,
+    typename Value,
+    typename Hash = std::hash<Key>,
+    typename Pred = std::equal_to<Key>,
+    typename Alloc = std::allocator<std::pair<const Key, Value>>>
+struct ConcurrentUnorderedMapTraits {
+   using key_type = Key;
+   using mapped_type = Value;
+   using hash_type = Hash;
+   using pred_type = Pred;
+   using alloc_type = Alloc;
+   using map_type = std::unordered_map<Key, Value, Hash, Pred, Alloc>;
+   using concurrent_map_type =
+       ConcurrentUnorderedMap<Key, Value, Hash, Pred, Alloc>;
+};
+
+template <
+    typename Key,
+    typename Value,
+    typename Hash,
+    typename Pred,
+    typename Alloc>
+class ConcurrentUnorderedMap {
+   std::mutex mx;
+   std::unordered_map<Key, Value> data;
+
+ public:
+   using Data = std::unordered_map<Key, Value, Hash, Pred, Alloc>;
+   using This = ConcurrentUnorderedMap<Key, Value, Hash, Pred, Alloc>;
+   using key_type = Data::key_type;
+   using value_type = Data::value_type;
+   using mapped_type = Data::mapped_type;
+   using hasher = Data::hasher;
+   using key_equal = Data::key_equal;
+   using allocator_type = Data::allocator_type;
+
+   using pointer = Data::pointer;
+   using const_pointer = Data::const_pointer;
+   using reference = Data::reference;
+   using const_reference = Data::const_reference;
+   using iterator =
+       ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc>;
+   using size_type = Data::size_type;
+   using difference_type = Data::difference_type;
+
+   bool empty() {
+      std::lock_guard<std::mutex> lock{mx};
+      return data.empty();
+   }
+
+   size_type size() {
+      std::lock_guard<std::mutex> lock{mx};
+      return data.size();
+   }
+
+   iterator begin() {
+      std::lock_guard<std::mutex> lock{mx};
+      return {data.begin(), mx};
+   }
+
+   iterator end() {
+      std::lock_guard<std::mutex> lock{mx};
+      return {data.end(), mx};
+   }
+
+   template <typename... Args>
+   std::pair<iterator, bool> emplace(Args &&...args) {
+      std::lock_guard<std::mutex> lock{mx};
+      auto [it, is_new] = data.emplace(std::forward<Args>(args)...);
+      return {iterator{it, mx}, is_new};
+   }
+
+   template <typename... Args>
+   std::pair<iterator, bool> try_emplace(const key_type &k, Args &&...args) {
+      std::lock_guard<std::mutex> lock{mx};
+      auto [it, is_new] = data.try_emplace(k, args...);
+      return {iterator{it, mx}, is_new};
+   }
+
+   std::pair<iterator, bool> insert(const value_type &x) {
+      std::lock_guard<std::mutex> lock{mx};
+      auto [it, is_new] = data.insert(x);
+      return {iterator{it, mx}, is_new};
+   }
+
+   template <typename Obj>
+   std::pair<iterator, bool> insert_or_assign(const key_type &k, Obj &&obj) {
+      std::lock_guard<std::mutex> lock{mx};
+      auto [it, is_new] = data.insert_or_assign(k, std::forward<Obj>(obj));
+      return {iterator{it, mx}, is_new};
+   }
+
+   iterator erase(iterator pos) {
+      std::lock_guard<std::mutex> lock{mx};
+      return {data.erase(pos.iter()), mx};
+   }
+
+   size_type erase(const key_type &k) {
+      std::lock_guard<std::mutex> lock{mx};
+      return data.erase(k);
+   }
+
+   void clear() {
+      std::lock_guard<std::mutex> lock{mx};
+      data.clear();
+   }
+
+   iterator find(const key_type &k) {
+      std::lock_guard<std::mutex> lock{mx};
+      return {data.find(k), mx};
+   }
+
+   bool contains(const key_type &k) {
+      std::lock_guard<std::mutex> lock{mx};
+      return data.contains(k);
+   }
+
+   mapped_type &at(const key_type &k) {
+      std::lock_guard<std::mutex> lock{mx};
+      return data.at(k);
+   }
+
+   mapped_type &operator[](const key_type &k) {
+      std::lock_guard<std::mutex> lock{mx};
+      return data[k];
+   }
+};
+
+} // namespace bts::collections

src/lib/collections/concurrent_unordered_map_iterator.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include <iterator>
+#include <mutex>
+#include <unordered_map>
+
+namespace bts::collections {
+
+template <
+    typename Key,
+    typename Value,
+    typename Hash = std::hash<Key>,
+    typename Pred = std::equal_to<Key>,
+    typename Alloc = std::allocator<std::pair<const Key, Value>>>
+class ConcurrentUnorderedMap_Iterator {
+   using iterator = std::unordered_map<Key, Value, Hash, Pred, Alloc>::iterator;
+   using iterator_category = std::forward_iterator_tag;
+   using value_type = std::pair<const Key, Value>;
+   using pointer = value_type *;
+   using reference = value_type &;
+   using This = ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc>;
+
+   iterator it;
+   std::mutex *mx_p;
+
+ public:
+   ConcurrentUnorderedMap_Iterator(iterator it, std::mutex &mx)
+       : it(it), mx_p(&mx) {}
+
+   ConcurrentUnorderedMap_Iterator(
+       const ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc>
+           &other)
+       : it(other.it), mx_p(other.mx_p) {}
+
+   ConcurrentUnorderedMap_Iterator(
+       ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc> &&other)
+       : it(std::move(other.it)), mx_p(other.mx_p) {}
+
+   ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc> &operator=(
+       const ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc>
+           &other) {
+      it = other.it;
+      mx_p = other.mx_p;
+      return *this;
+   }
+
+   ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc> &operator=(
+       ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc> &&other) {
+      it = std::move(other.it);
+      mx_p = other.mx_p;
+      return *this;
+   }
+
+   iterator &iter() { return it; }
+
+   reference operator*() {
+      std::lock_guard<std::mutex> lock{*mx_p};
+      return *it;
+   }
+
+   pointer operator->() {
+      std::lock_guard<std::mutex> lock{*mx_p};
+      return it.operator->();
+   }
+
+   This &operator++() {
+      std::lock_guard<std::mutex> lock{*mx_p};
+      This before{it, *mx_p};
+      ++it;
+      return *this;
+   }
+
+   This operator++(int) {
+      std::lock_guard<std::mutex> lock{*mx_p};
+      This before{it, *mx_p};
+      ++it;
+      return before;
+   }
+
+   bool operator==(
+       const ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc>
+           &other) {
+      std::lock_guard<std::mutex> lock{*mx_p};
+      return it == other.it;
+   }
+   bool operator!=(
+       const ConcurrentUnorderedMap_Iterator<Key, Value, Hash, Pred, Alloc>
+           &other) {
+      std::lock_guard<std::mutex> lock{*mx_p};
+      return it != other.it;
+   }
+};
+
+} // namespace bts::collections

src/lib/collections/tests/concurrent_unordered_map_tests.cpp

@@ -0,0 +1,155 @@
+#include "lib/collections/concurrent_unordered_map.h"
+#include "lib/collections/concurrent_unordered_map_iterator.h"
+
+#include <catch2/catch_all.hpp>
+#include <chrono>
+#include <string>
+#include <thread>
+
+namespace bts::collections::tests {
+
+template <typename Traits>
+static bool require_map_equals(
+    typename Traits::concurrent_map_type &map,
+    const typename Traits::map_type &expected_map) {
+   REQUIRE(map.size() == expected_map.size());
+   for (auto &[k, v] : map) {
+      auto it = expected_map.find(k);
+      REQUIRE(it != expected_map.end() && it->second == v);
+   }
+}
+
+template <typename Traits>
+static bool map_equals(
+    typename Traits::concurrent_map_type &map,
+    const typename Traits::map_type &expected_map) {
+   if (map.size() != expected_map.size()) {
+      return false;
+   }
+   for (auto &[k, v] : map) {
+      auto it = expected_map.find(k);
+      if (it == expected_map.end() || it->second != v) {
+         return false;
+      }
+   }
+   return true;
+}
+
+struct ConcurrentInsertTestContext {
+   ConcurrentUnorderedMap<std::string, int> map;
+   const std::vector<std::pair<std::string, int>> items1;
+   const std::vector<std::pair<std::string, int>> items2;
+   std::unordered_map<std::string, int> expected_map;
+
+   ConcurrentInsertTestContext(
+       std::vector<std::pair<std::string, int>> items1,
+       std::vector<std::pair<std::string, int>> items2)
+       : items1(std::move(items1)), items2(std::move(items2)) {}
+
+   void run() {
+      expected_map.clear();
+      for (const auto &item : this->items1) {
+         expected_map.insert(item);
+      }
+      for (const auto &item : this->items2) {
+         expected_map.insert(item);
+      }
+      std::atomic<int> i1 = 0;
+      std::atomic<int> i2 = 0;
+      std::thread t1{[&]() { run_impl(items1, i1, i2); }};
+      std::thread t2{[&]() { run_impl(items2, i2, i1); }};
+      t1.join();
+      t2.join();
+   }
+
+ private:
+   void run_impl(
+       const std::vector<std::pair<std::string, int>> &items,
+       std::atomic<int> &loop,
+       std::atomic<int> &other_loop) {
+      static constexpr const int LOOP_COUNT = 1000;
+      static constexpr const int MAX_ITER_DIFF = 100;
+      for (; loop < LOOP_COUNT; loop++) {
+         for (const auto &[k, v] : items) {
+            map[k] = v;
+         }
+         // Let other thread catch up
+         for (int i = 0; (loop - other_loop) > MAX_ITER_DIFF; i++) {
+            if (i % 100 == 0) {
+               std::this_thread::sleep_for(std::chrono::microseconds(100));
+            } else {
+               std::this_thread::yield();
+            }
+         }
+         if ((loop - other_loop) > MAX_ITER_DIFF) {
+            break;
+         }
+      }
+   }
+};
+
+TEST_CASE("ConcurrentUnorderedMapTests", "[collection_tests]") {
+   SECTION("insert") {
+      ConcurrentUnorderedMap<std::string, int> map;
+      map["foo"] = 4;
+      REQUIRE(map.size() == 1);
+      REQUIRE(map.contains("foo"));
+      REQUIRE(map["foo"] == 4);
+      auto it = map.find("foo");
+      REQUIRE((it != map.end() && it->second == 4));
+   }
+   SECTION("update") {
+      ConcurrentUnorderedMap<std::string, int> map;
+      map["foo"] = 4;
+      map["foo"] = 5;
+      REQUIRE(map["foo"] == 5);
+   }
+   SECTION("emplace") {
+      ConcurrentUnorderedMap<std::string, int> map;
+      const std::string name = "foo";
+      auto res = map.try_emplace(name, 4);
+      REQUIRE(res.second);
+      REQUIRE((res.first->first == name && res.first->second == 4));
+      res = map.try_emplace(name, 5);
+      REQUIRE(!res.second);
+      REQUIRE((res.first->first == name && res.first->second == 4));
+      map.clear();
+      std::unordered_map<std::string, int> um;
+      um.emplace(name, 4);
+      res = map.emplace(name, 4);
+      REQUIRE(map[name] == 4);
+   }
+   SECTION("concurrent_insert") {
+      ConcurrentInsertTestContext ctx{
+          {
+              {"a", 1},
+              {"b", 2},
+              {"c", 3},
+              {"d", 4},
+              {"e", 5},
+              {"f", 6},
+              {"g", 7},
+              {"h", 8},
+              {"i", 9},
+              {"j", 10},
+          },
+          // Some values overlap, some don't.
+          {
+              {"A", 1},
+              {"b", 2},
+              {"C", 3},
+              {"d", 4},
+              {"E", 5},
+              {"f", 6},
+              {"G", 7},
+              {"h", 8},
+              {"I", 9},
+              {"j", 10},
+          }};
+      ctx.run();
+      REQUIRE(map_equals<ConcurrentUnorderedMapTraits<std::string, int>>(
+          ctx.map, ctx.expected_map));
+   }
+}
+
+} // namespace bts::collections::tests

src/lib/config/profiles/md_dev/market_data_config.conf

@@ -77,7 +77,7 @@ MarketDataConfig {
     okx {
       num_depth_levels: 5
       url_rest_base: "https://aws.okx.com"
-      url_ws_base: "wss://wsaws.okx.com:8443"
+      url_ws_base: "wss://ws.okx.com:8443"
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_SPOT

src/lib/core/start_stop_state_machine.cpp

@@ -0,0 +1,211 @@
+#include "start_stop_state_machine.h"
+#include "lib/io/json_formatter.h"
+#include "lib/logging/Common.h"
+#include "lib/logging/logging.h"
+#include "lib/logging/trace_log_var.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/bts_lock_guard.h"
+
+namespace bts::core {
+
+using std::int64_t;
+using std::string;
+
+#define INFO_TRACE(fn)                                                         \
+   LOG_TRACE_VAR_PARAMS(                                                       \
+       subsystem_, Info, name_, string{fn},                                    \
+       bts::io::json_formatter(state_name()))
+
+StartStopStateMachineBase::~StartStopStateMachineBase() = default;
+
+const std::string &StartStopStateMachineBase::state_name() const {
+   switch (state_) {
+   case IDLE: {
+      static const std::string s{"IDLE"};
+      return s;
+   }
+   case STARTING: {
+      static const std::string s{"STARTING"};
+      return s;
+   }
+   case WAITING_TO_START: {
+      static const std::string s{"WAITING_TO_START"};
+      return s;
+   }
+   case RUNNING: {
+      static const std::string s{"RUNNING"};
+      return s;
+   }
+   case WAITING_TO_STOP: {
+      static const std::string s{"WAITING_TO_STOP"};
+      return s;
+   }
+   case STOPPING: {
+      static const std::string s{"STOPPING"};
+      return s;
+   }
+   default: {
+      static const std::string s{"UNKNOWN"};
+      return s;
+   }
+   }
+}
+
+void StartStopStateMachineBase::start_core() {}
+void StartStopStateMachineBase::stop_core() {}
+
+int64_t StartStopStateMachineBase::start() {
+   INFO_TRACE("start");
+   int64_t vers;
+   {
+      BTS_LOCK_GUARD(mx_);
+      switch (state_) {
+      case RUNNING:
+      case STARTING:
+      case WAITING_TO_START:
+         return version_;
+      case STOPPING:
+         // Don't increment version until we initiate START.
+         state_ = WAITING_TO_START;
+         return version_;
+      case WAITING_TO_STOP:
+         // Cancel pending stop.
+         state_ = STARTING;
+         return version_;
+      case IDLE:
+         // Initiate start and increment version.
+         state_ = STARTING;
+         vers = ++version_;
+         break;
+      default:
+         bts_assert_msgf(
+             false, "{} invalid internal state '{}'", name_, state_name());
+         return version_;
+      }
+   }
+   start_core();
+   bool stop_requested = false;
+   {
+      BTS_LOCK_GUARD(mx_);
+      switch (state_) {
+      case STARTING:
+         break;
+      case WAITING_TO_STOP:
+         stop_requested = true;
+         break;
+      default:
+         bts_assert_msgf(false, "Invalid internal state '{}'", state_name());
+         return vers;
+      }
+      vers = ++version_;
+      state_ = RUNNING;
+   }
+   if (stop_requested) {
+      vers = stop();
+   }
+   return vers;
+}
+
+int64_t StartStopStateMachineBase::stop() {
+   INFO_TRACE("stop");
+   int64_t vers;
+   {
+      BTS_LOCK_GUARD(mx_);
+      switch (state_) {
+      case IDLE:
+      case STOPPING:
+      case WAITING_TO_STOP:
+         return version_;
+      case STARTING:
+         // Don't increment version until we initiate STOP.
+         state_ = WAITING_TO_STOP;
+         return version_;
+      case WAITING_TO_START:
+         // Cancel pending start.
+         state_ = STOPPING;
+         return version_;
+      case RUNNING:
+         // Initiate stop and increment version.
+         state_ = STOPPING;
+         vers = ++version_;
+         break;
+      default:
+         bts_assert_msgf(
+             false, "{} invalid internal state '{}'", name_, state_name());
+         return version_;
+      }
+   }
+   stop_core();
+   bool start_requested = false;
+   {
+      BTS_LOCK_GUARD(mx_);
+      switch (state_) {
+      case STOPPING:
+         break;
+      case WAITING_TO_START:
+         start_requested = true;
+         break;
+      default:
+         bts_assert_msgf(
+             false, "{} invalid internal state '{}'", name_, state_name());
+         break;
+      }
+      state_ = IDLE;
+   }
+   if (start_requested) {
+      vers = start();
+   }
+   return vers;
+}
+
+StartStopStateMachine::StartStopStateMachine(
+    const std::string &name, log::Subsystem subsystem) {
+   set_name(name);
+   set_subsystem(subsystem);
+}
+
+StartStopStateMachineBase::State StartStopStateMachineBase::state() {
+   BTS_LOCK_GUARD(mx_);
+   return state_;
+}
+
+int64_t StartStopStateMachineBase::version() {
+   BTS_LOCK_GUARD(mx_);
+   return version_;
+}
+
+bool StartStopStateMachineBase::is_stopping() {
+   BTS_LOCK_GUARD(mx_);
+   return state_ == STOPPING;
+}
+
+void StartStopStateMachineBase::invoke_if(
+    State states, std::function<void(State)> fn) {
+   BTS_LOCK_GUARD(mx_);
+   if ((state_ & states) == state_) {
+      fn(state_);
+   }
+}
+
+void StartStopStateMachineBase::invoke_if_active(
+    std::function<void(State)> fn) {
+   BTS_LOCK_GUARD(mx_);
+   switch (state_) {
+   case STARTING:
+   case RUNNING:
+      fn(state_);
+      break;
+   default:
+      break;
+   }
+}
+
+void StartStopStateMachineBase::invoke(std::function<void(State)> fn) {
+   BTS_LOCK_GUARD(mx_);
+   fn(state_);
+}
+
+void StartStopStateMachine::start_core() { start_core_impl(); }
+void StartStopStateMachine::stop_core() { stop_core_impl(); }
+
+} // namespace bts::core

src/lib/core/start_stop_state_machine.h

@@ -0,0 +1,100 @@
+#pragma once
+
+#include "lib/logging/Common.h"
+#include "lib/utils/Delegate.h"
+
+#include <functional>
+#include <mutex>
+#include <string>
+
+namespace bts::core {
+
+struct StartStopStateMachineBase {
+   virtual ~StartStopStateMachineBase();
+
+   const std::string &name() const;
+   log::Subsystem subsystem() const;
+   void set_name(const std::string &value);
+   void set_subsystem(log::Subsystem value);
+
+   std::int64_t start();
+   std::int64_t stop();
+   std::int64_t version();
+   std::recursive_mutex &mutex();
+
+   enum State {
+      IDLE = 0,
+      STARTING = 1,
+      RUNNING = 1 << 1,
+      STOPPING = 1 << 2,
+      WAITING_TO_START = 1 << 3,
+      WAITING_TO_STOP = 1 << 4,
+   };
+
+   State state();
+   const std::string &state_name() const;
+
+   bool is_stopping();
+   void invoke_if(State states, std::function<void(State)> fn);
+   void invoke_if_active(std::function<void(State)> fn);
+   void invoke(std::function<void(State)> fn);
+
+ protected:
+   virtual void start_core();
+   virtual void stop_core();
+
+ private:
+   std::string name_;
+   log::Subsystem subsystem_;
+   std::int64_t version_;
+
+   State state_ = State::IDLE;
+   std::recursive_mutex mx_;
+};
+
+struct StartStopStateMachine : public StartStopStateMachineBase {
+   StartStopStateMachine() = default;
+   StartStopStateMachine(const std::string &name, log::Subsystem subsystem);
+
+   bts::core::Delegate<void()> start_core_impl;
+   bts::core::Delegate<void()> stop_core_impl;
+
+ protected:
+   void start_core() override;
+   void stop_core() override;
+};
+
+constexpr StartStopStateMachineBase::State operator|(
+    StartStopStateMachineBase::State lhs,
+    StartStopStateMachineBase::State rhs) noexcept {
+   return static_cast<StartStopStateMachineBase::State>(
+       static_cast<int>(lhs) | static_cast<int>(rhs));
+}
+constexpr StartStopStateMachineBase::State operator&(
+    StartStopStateMachineBase::State lhs,
+    StartStopStateMachineBase::State rhs) noexcept {
+   return static_cast<StartStopStateMachineBase::State>(
+       static_cast<int>(lhs) & static_cast<int>(rhs));
+}
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline const std::string &StartStopStateMachineBase::name() const {
+   return name_;
+}
+inline void StartStopStateMachineBase::set_name(const std::string &value) {
+   name_ = value;
+}
+inline log::Subsystem StartStopStateMachineBase::subsystem() const {
+   return subsystem_;
+}
+inline void StartStopStateMachineBase::set_subsystem(
+    bts::log::Subsystem value) {
+   subsystem_ = value;
+}
+
+inline std::recursive_mutex &StartStopStateMachineBase::mutex() { return mx_; }
+
+} // namespace bts::core

src/lib/io/websocket/websocket_client.cpp

@@ -11,6 +11,7 @@
 #include "lib/logging/logging.h"
 #include "lib/utils/LockBox.h"
 #include "lib/utils/bts_assert.h"
+#include "lib/utils/bts_lock_guard.h"
 #include "lib/utils/formatters/exception_formatter.h"
 #include "lib/utils/thread_manager.h"
 
@@ -105,8 +106,8 @@ using bts::threading::configure_current_thread;
 using nlohmann::json;
 using std::lock_guard;
 using std::map;
-using std::mutex;
 using std::optional;
+using std::recursive_mutex;
 using std::size_t;
 using std::string;
 using std::thread;
@@ -156,7 +157,7 @@ struct WebsocketClient::PImpl {
    optional<thread> ping_thread;
    utils::LockBox<vector<thread>> io_threads;
    int client_id{client_id_counter++};
-   mutex instance_mx;
+   recursive_mutex instance_mx;
    State state = State::IDLE;
    int version = 0;
    int n_reconnects = 0;
@@ -231,7 +232,7 @@ struct WebsocketClient::PImpl {
    }
 
    void close(websocketpp::close::status::value code, std::string reason) {
-      lock_guard<mutex> lock{instance_mx};
+      BTS_LOCK_GUARD(instance_mx);
       INFO_FN(
           "close. {}; code: {}; reason: {}", snapshot(),
           WebsocketCloseStatus{code}, reason);
@@ -272,7 +273,7 @@ struct WebsocketClient::PImpl {
     * Connects to the websocket server
     */
    void connect() {
-      lock_guard<mutex> lock{instance_mx};
+      BTS_LOCK_GUARD(instance_mx);
       INFO_FN("connect. {}", snapshot());
       switch (state.value) {
       case State::IDLE:
@@ -421,7 +422,7 @@ struct WebsocketClient::PImpl {
    }
 
    void on_open(ConnectionHdl hdl) {
-      lock_guard<mutex> lock{instance_mx};
+      BTS_LOCK_GUARD(instance_mx);
       INFO_FN("on_open. {}", snapshot());
       const auto ec = client.get_con_from_hdl(hdl)->get_ec();
       if (ec) {
@@ -454,7 +455,7 @@ struct WebsocketClient::PImpl {
    }
 
    void on_close(ConnectionHdl hdl) {
-      lock_guard<mutex> lock{instance_mx};
+      BTS_LOCK_GUARD(instance_mx);
       INFO_FN("on_close. {}", snapshot());
       switch (state.value) {
       case State::CLOSING:
@@ -483,7 +484,7 @@ struct WebsocketClient::PImpl {
    }
 
    void on_fail(ConnectionHdl hdl) {
-      lock_guard<mutex> lock{instance_mx};
+      BTS_LOCK_GUARD(instance_mx);
       ERROR_FN("on_fail. {}", snapshot());
       set_state(State::IDLE);
       const auto con = client.get_con_from_hdl(hdl);
@@ -558,7 +559,7 @@ void WebsocketClient::disconnect() {
 }
 
 bool WebsocketClient::is_connected() const {
-   lock_guard<mutex> lock{me_->instance_mx};
+   lock_guard<recursive_mutex> lock{me_->instance_mx};
    return me_->state == State::OPEN;
 }
 

src/lib/logging/log_var.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "lib/logging/logging.h"
+
+#define LOG_VAR(subsystem, level, ...)                                         \
+   bts::log::_thread_logger.logger->is_loggable(subsystem, level) &&           \
+       bts::log::_thread_logger.logger->enqueue(                               \
+           bts::log::log_ts(), subsystem, level, __FILE__, __LINE__,           \
+           __VA_ARGS__)

src/lib/logging/trace_log_var.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "lib/logging/log_var.h"
+
+#define LOG_TRACE_VAR(subsystem, level, scope, name)                                 \
+   LOG_VAR(                                                                          \
+       subsystem, level,                                                             \
+       R"([trace.json]{{ "scope": "{}", "name": "{}", "params": null, "ts": {} }})", \
+       scope, name, static_cast<std::int64_t>(bts::log::log_ts()))
+
+#define LOG_TRACE_VAR_PARAMS(subsystem, level, scope, name, params)                \
+   LOG_VAR(                                                                        \
+       subsystem, level,                                                           \
+       R"([trace.json]{{ "scope": "{}", "name": "{}", "params": {}, "ts": {} }})", \
+       scope, name, params, static_cast<std::int64_t>(bts::log::log_ts()))

src/lib/orderman/gateways/OkxExecutionGateway.cpp

@@ -1,4 +1,5 @@
 #include "OkxExecutionGateway.h"
+#include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/config/protos/okx_exe_cfg.pb.h"
 #include "lib/domain_model/Security.pb.h"
@@ -12,6 +13,7 @@
 #include "lib/orderman/CancelOrderMsg.h"
 #include "lib/orderman/CancelReplaceOrderMsg.h"
 #include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/events/NewAdminDataEvent.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/orderman/orderman_utils/ExchangePosition.h"
 #include "lib/orderman/orderman_utils/ExecutionReport.h"
@@ -168,6 +170,7 @@ struct OkxExecutionGateway::PImpl {
        config::get<config::OkxExeCfg>().ws_ping_interval_seconds());
    websocket::WebsocketClient ws_client;
    tbb::concurrent_queue<std::shared_ptr<ExecutionReport>> buffered_exec_reps;
+   tbb::concurrent_queue<ExchangePosition> exch_pos_queue;
    std::optional<std::thread> ws_thread;
    OnExeReport cb;
    OnGwStatusChange on_gw_status_change;
@@ -178,6 +181,7 @@ struct OkxExecutionGateway::PImpl {
    std::shared_ptr<const ID_GEN> id_gen = std::make_shared<const ID_GEN>();
    std::atomic<OkxExecutionState> state = OkxExecutionState::OFF;
    ExchangePosition exchange_position;
+
    PImpl(md::SecurityOracle o)
        : ws_client(websocket::WebsocketClient(
              get_ws_client_config(url, connect_timeout, pong_timeout))),
@@ -310,34 +314,30 @@ struct OkxExecutionGateway::PImpl {
    std::optional<std::thread> pos_polling_thread;
    std::atomic<bool> pos_polling_running{false};
    void position_polling();
-   void start_pos_polling() {
+   void start_pos_polling(
+       const uint32_t account_id, const std::string &account_name) {
       if (pos_polling_running)
          return;
       pos_polling_running = true;
-      pos_polling_thread.emplace([this]() {
-         ExchangePosition positions;
+      pos_polling_thread.emplace([this, account_name, account_id]() {
+         // ExchangePosition positions;
          auto venue = api::VENUE_OKX;
-         positions.exchange = venue;
-         auto sleep_ms = 2000ms;
+         // positions.exchange = venue;
+         auto sleep_ms = 5000ms;
 
          //create positions request
          const std::string REQUEST_PATH = "/api/v5/account/positions";
-         
-         //data for positions
-         const std::string account_name =
-         config::get<config::OkxExeCfg>().account();
-         
+
          while (pos_polling_running) {
-             
-             http::HttpResponse res;
-             auto headers = OkxUtils::createHeaders(
-                 api_key, passphrase, secret_key, "GET", REQUEST_PATH,
-                 simulated_trading, "");
+            ExchangePosition positions = {venue, {}};
+            http::HttpResponse res;
+            auto headers = OkxUtils::createHeaders(
+                api_key, passphrase, secret_key, "GET", REQUEST_PATH,
+                simulated_trading, "");
             http::HttpClient::get(
                 config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH,
                 headers, &res);
-                LOG(OrderMan, Info, "pos poll response: {}", res.body);
-            positions.total_exchange_positions.clear();
+            LOG(OrderMan, Info, "pos poll response: {}", res.body);
             auto body = OkxUtils::parseJsonSafely(res.body);
             if (body && body->contains("data") && (*body)["data"].is_array()) {
                auto &j = *body;
@@ -370,10 +370,15 @@ struct OkxExecutionGateway::PImpl {
                      avgPx = std::stod(item["avgPx"].get<std::string>());
                   }
                   positions.total_exchange_positions.push_back(
-                      {instId, sec, account_name, std::nullopt, pos,
-                       Price(avgPx)});
+                      ExchangePosition::UnitPosition{
+                          instId, sec, account_name, account_id, pos,
+                          Price(avgPx)});
+               }
+               if (on_exchange_position_state)
+                  exch_pos_queue.push(std::move(positions));
+               else {
+                  LOG(OrderMan, Info, "on_exchange_position_state not defined");
                }
-               on_exchange_position_state(positions);
             }
             std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
          }
@@ -490,7 +495,28 @@ struct OkxExecutionGateway::PImpl {
          start_impl();
       });
    }
-
+   void send_pos() {
+      if (exch_pos_queue.empty()) {
+         return;
+      }
+      ExchangePosition pos;
+      while (exch_pos_queue.try_pop(pos)) {
+         on_exchange_position_state(std::move(pos));
+      }
+   }
+   void send_exes() {
+      if (buffered_exec_reps.empty()) {
+         return;
+      }
+      std::shared_ptr<ExecutionReport> er;
+      while (buffered_exec_reps.try_pop(er)) {
+         trace::CorID cid = {};
+         LOG(OrderMan, Info,
+             "OkxExecutionGateway::dispatch_incoming_fix_messages er: {} ",
+             er->to_string());
+         cb(std::move(er), cid);
+      }
+   }
    void start_impl() {
       size_t attempt = 0;
       while (state != OkxExecutionState::OFF) {
@@ -685,18 +711,36 @@ void OkxExecutionGateway::modify_order(
       OkxUtils::handleOkException("Amend order", err);
    }
 }
-void OkxExecutionGateway::dispatch_incoming_fix_messages() {
-   if (me_->buffered_exec_reps.empty()) {
+void OkxExecutionGateway::on_admin_data(
+    const bts::mercury::Update<NewAdminDataEvent> &update) {
+   LOG(OrderMan, Info, "OkxExecutionGateway::on_admin_data t_id: {} t_name: {}",
+       bts::os::get_thread_id(), bts::os::get_thread_name());
+   if (update.events == nullptr) {
       return;
    }
-   std::shared_ptr<ExecutionReport> er;
-   while (me_->buffered_exec_reps.try_pop(er)) {
-      trace::CorID cid = {};
-      LOG(OrderMan, Info,
-          "OkxExecutionGateway::dispatch_incoming_fix_messages er: {} ",
-          er->to_string());
-      me_->cb(std::move(er), cid);
+   const auto *account = bts::admin::AdminServiceUtils::find_account(
+       *update.events->back().admin_data,
+       config::get<config::OkxExeCfg>().account());
+   if (account == nullptr) {
+      LOG(OrderMan, Warn,
+          "OkxExecutionGateway::on_admin_data Gateway account not found. "
+          "Cannot execute orders. name: '{}'",
+          config::get<config::OkxExeCfg>().account());
+      return;
    }
+   if (config::get<config::OkxExeCfg>().account() != account->name() ||
+       me_->pos_polling_running) {
+      return;
+   }
+   me_->start_pos_polling(account->id(), account->name());
+   LOG(OrderMan, Info,
+       "OkxExecutionGateway::on_admin_data Resolved gateway account '{}', pos "
+       "polling started",
+       account->name());
+};
+void OkxExecutionGateway::dispatch_incoming_fix_messages() {
+   me_->send_exes();
+   me_->send_pos();
 }
 OnExeReport &OkxExecutionGateway::on_exe_report() { return me_->cb; }
 OnCancelReject &OkxExecutionGateway::on_cancel_reject() {
@@ -726,11 +770,17 @@ void OkxExecutionGateway::launch_new_order(const CreateOrderMsg &msg) {
       auto pos = ord_full_name.find(suffix);
       std::string order_type;
       switch (msg.time_in_force) {
-         case api::orderman::TIME_IN_FORCE_IOC: order_type = "ioc"; break;
-         case api::orderman::TIME_IN_FORCE_FOK: order_type = "fok"; break;
-         default: order_type = FIX::string_toLower(ord_full_name.substr(pos + suffix.size()));
+      case api::orderman::TIME_IN_FORCE_IOC:
+         order_type = "ioc";
+         break;
+      case api::orderman::TIME_IN_FORCE_FOK:
+         order_type = "fok";
+         break;
+      default:
+         order_type =
+             FIX::string_toLower(ord_full_name.substr(pos + suffix.size()));
       }
-      const std::string ord_type =order_type;
+      const std::string ord_type = order_type;
 
       const std::string body = fmt::format(
           body_template, msg.security->okx().security_desc(), me_->tdmode, side,

src/lib/orderman/gateways/OkxExecutionGateway.h

@@ -15,9 +15,7 @@ class OkxExecutionGateway : public IExecutionGateway {
    void on_securities(const mercury::Update<NewSecuritiesEvent> &) override {
     LOG(md, Info,"Not implemented");
    };
-   void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override{
-    LOG(md, Info,"Not implemented");
-   };
+   void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
    void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override{
     LOG(md, Info,"Not implemented");
    }

src/lib/utils/observable_value_store.h

@@ -0,0 +1,93 @@
+#pragma once
+
+#include "lib/utils/Delegate.h"
+
+#include <functional>
+#include <optional>
+
+namespace bts::utils {
+
+template <typename T, typename Comp = std::equal_to<T>>
+struct ObservableValueStore {
+   ObservableValueStore() = default;
+   ObservableValueStore(T initial_value, Comp comp)
+       : value_(std::move(initial_value)), comp_(comp) {}
+
+   const T &value() const { return value_; }
+
+   void set_value(T value) {
+      if (comp_(value_, value)) {
+         return;
+      }
+      if (value_changing_) {
+         (*value_changing_)(value_, value);
+      }
+      value_ = std::move(value);
+      if (value_changed_) {
+         (*value_changed_)(value_);
+      }
+   }
+
+   /**
+    * Callback with old and new value.
+    */
+   core::Delegate<void(const T &, const T &)> &value_changing() {
+      if (!value_changing_) {
+         value_changing_.emplace();
+      }
+      return *value_changing_;
+   }
+
+   /**
+    * Callback with new value.
+    */
+   core::Delegate<void(const T &)> &value_changed() {
+      if (!value_changed_) {
+         value_changed_.emplace();
+      }
+      return *value_changed_;
+   }
+
+ private:
+   std::optional<core::Delegate<void(const T &, const T &)>> value_changing_;
+   std::optional<core::Delegate<void(const T &)>> value_changed_;
+   T value_;
+   Comp comp_;
+};
+
+template <typename T, typename Comp = std::equal_to<T>>
+struct FunctionalObservableValueStore {
+   FunctionalObservableValueStore(std::function<T()> get_value)
+       : get_value_(get_value) {}
+
+   FunctionalObservableValueStore(std::function<T()> get_value, T initial_value)
+       : get_value_(get_value), store_(initial_value, {}) {}
+
+   FunctionalObservableValueStore(
+       std::function<T()> get_value, T initial_value, Comp comp)
+       : get_value_(get_value), store_(initial_value, comp) {}
+
+   const T &value() { return store_.value(); }
+
+   /**
+    * Callback with old and new value.
+    */
+   core::Delegate<void(const T &, const T &)> &value_changing() {
+      return store_.value_changing();
+   }
+
+   /**
+    * Callback with new value.
+    */
+   core::Delegate<void(const T &)> &value_changed() {
+      return store_.value_changed();
+   }
+
+   void refresh() { store_.set_value(get_value_()); }
+
+ private:
+   std::function<T()> get_value_;
+   ObservableValueStore<T, Comp> store_;
+};
+
+} // namespace bts::utils

src/lib_binance_execution/binance_execution/execution_report_store.cpp

@@ -4,6 +4,8 @@
 #include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include "lib/utils/string_utils.h"
 
+#include <limits>
+
 namespace binance_execution {
 
 using bts::orderman::ExecutionReport;

src/lib_binance_execution/binance_execution/position_calculator.cpp

@@ -0,0 +1,157 @@
+#include "position_calculator.h"
+#include "binance_execution/enums/position_side.h"
+#include "lib/api/services/OrderService.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/md/SecurityOracle.h"
+#include "lib/orderman/orderman_utils/ExchangePosition.h"
+#include "lib/utils/bts_assert.h"
+
+#include <unordered_map>
+
+namespace binance_execution {
+
+using bts::md::SecurityOracle;
+using std::int64_t;
+using std::string;
+using std::uint32_t;
+using std::unordered_map;
+
+struct PositionSideData {
+   double quantity;
+   double avg_cost;
+   std::int64_t timestamp;
+};
+
+struct PositionData {
+   PositionSideData lng;
+   PositionSideData sht;
+};
+
+struct PositionCalculator::PImpl {
+   string account;
+   uint32_t account_id;
+   bts::api::Venue venue;
+   bts::api::SecurityType type;
+   SecurityOracle sec_oracle;
+   // data mapped as Symbol -> pair(Long, Short)
+   unordered_map<string, PositionData> data;
+
+   PImpl(
+       string account,
+       uint32_t account_id,
+       bts::api::Venue venue,
+       bts::api::SecurityType type,
+       SecurityOracle sec_oracle)
+       : account(std::move(account)), account_id(account_id), venue(venue),
+         type(type), sec_oracle(std::move(sec_oracle)) {}
+
+   PositionData &get_pos_data(const string &symbol) {
+      return data.try_emplace(symbol).first->second;
+   }
+
+   PositionSideData &get_pos_data(const string &symbol, PositionSide side) {
+      auto &pos = data.try_emplace(symbol).first->second;
+      switch (side.value) {
+      case PositionSide::LONG:
+         return pos.lng;
+      case PositionSide::SHORT:
+         return pos.sht;
+      default:
+         bts_assert_msg(false, "Expected PositionSide LONG or SHORT");
+         // gotta return something
+         return pos.lng;
+      }
+   }
+};
+
+PositionCalculator::PositionCalculator(
+    string account,
+    uint32_t account_id,
+    bts::api::Venue venue,
+    bts::api::SecurityType type,
+    SecurityOracle sec_oracle)
+    : me(std::make_unique<PImpl>(
+          std::move(account), account_id, venue, type, std::move(sec_oracle))) {
+}
+
+PositionCalculator::~PositionCalculator() = default;
+
+bool PositionCalculator::on_update(
+    const string &symbol,
+    PositionSide side,
+    double quantity,
+    double cost,
+    int64_t timestamp,
+    bts::orderman::ExchangePosition *pos_out) {
+   auto &me = *this->me;
+   auto &pos = me.get_pos_data(symbol);
+   switch (side.value) {
+   case PositionSide::NONE:
+      break;
+   case PositionSide::LONG:
+      pos.lng.timestamp = timestamp;
+      pos.lng.avg_cost = cost;
+      pos.lng.quantity = quantity;
+      break;
+   case PositionSide::SHORT:
+      pos.sht.timestamp = timestamp;
+      pos.sht.avg_cost = cost;
+      pos.sht.quantity = -quantity;
+      break;
+   case PositionSide::BOTH:
+      if (quantity > 0) {
+         pos.lng.timestamp = timestamp;
+         pos.lng.avg_cost = cost;
+         pos.lng.quantity = quantity;
+      } else {
+         pos.sht.timestamp = timestamp;
+         pos.sht.avg_cost = cost;
+         pos.sht.quantity = -quantity;
+      }
+      break;
+   }
+   if (pos_out == nullptr) {
+      return true;
+   }
+   auto &dest = pos_out->total_exchange_positions;
+   auto it = std::lower_bound(
+       dest.begin(), dest.end(),
+       std::pair<uint32_t, const string &>{me.account_id, symbol},
+       [](const bts::orderman::ExchangePosition::UnitPosition &u,
+          std::pair<uint32_t, const string &> p) {
+          if (u.account_id != p.first) {
+             return u.account_id < p.first;
+          }
+          return u.exchange_sec_desc < p.second;
+       });
+   if (it == dest.end() || it->exchange_sec_desc != symbol) {
+      const auto *sec = me.sec_oracle.get_security(me.venue, me.type, symbol);
+      bts::orderman::ExchangePosition::UnitPosition new_pos;
+      new_pos.exchange_sec_desc = symbol;
+      new_pos.security = sec;
+      new_pos.account_id = me.account_id;
+      new_pos.account_name = me.account;
+      it = dest.insert(it, new_pos);
+   }
+   switch (side.value) {
+   case PositionSide::BOTH:
+      it->position = quantity;
+      it->avg_filled_price.value() =
+          quantity > 0 ? pos.lng.avg_cost : pos.sht.avg_cost;
+      break;
+   case PositionSide::SHORT:
+      it->position = -quantity;
+      it->avg_filled_price.value() = pos.sht.avg_cost;
+      break;
+   case PositionSide::LONG:
+      it->position = quantity;
+      it->avg_filled_price.value() = pos.lng.avg_cost;
+      break;
+   default:
+      bts_assert_msgf(false, "Invalid PositionSide '{}'", side);
+      break;
+   }
+   return true;
+}
+
+} // namespace binance_execution

src/lib_binance_execution/binance_execution/position_calculator.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "binance_execution/enums/position_side.h"
+#include "lib/domain_model/Security.pb.h"
+
+namespace bts::md {
+class SecurityOracle;
+}
+
+namespace bts::orderman {
+class ExchangePosition;
+} // namespace bts::orderman
+
+namespace binance_execution {
+
+class PositionCalculator {
+ public:
+   PositionCalculator(
+       std::string account,
+       std::uint32_t account_id,
+       bts::api::Venue venue,
+       bts::api::SecurityType type,
+       bts::md::SecurityOracle sec_oracle);
+   ~PositionCalculator();
+
+   bool on_update(
+       const std::string &symbol,
+       PositionSide side,
+       double quantity,
+       double cost,
+       std::int64_t timestamp,
+       bts::orderman::ExchangePosition *pos_out = nullptr);
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+};
+
+} // namespace binance_execution

src/lib_binance_execution/binance_execution/price.h

@@ -8,7 +8,7 @@ namespace binance_execution {
 void price_to_string(
     double value,
     std::string &dest,
-    int n_decimals = 4,
+    int n_decimals = 5,
     std::size_t capacity = 24);
 
 //////////////////////////////////////////////////

src/lib_binance_execution/binance_execution/spots/messages/create_order_request.cpp

@@ -42,7 +42,6 @@ void CreateOrderRequest::to_signature_params(std::string &payload) const {
       payload.append("&newOrderRespType=");
       payload.append(newClientOrderId);
    }
-   // Format price & qty with 4 decimal places (should be sufficient the contracts we support)
    if (!std::isnan(price)) {
       payload.append("&price=");
       price_to_string(price, payload);

src/lib_binance_execution/binance_execution/usds_futures/messages/create_order_request.cpp

@@ -66,7 +66,6 @@ void CreateOrderRequest::to_signature_params(std::string &payload) const {
       payload.append("&positionSide=");
       payload.append(positionSide.name());
    }
-   // Format price & qty with 4 decimal places (should be sufficient the contracts we support)
    if (!std::isnan(price)) {
       payload.append("&price=");
       price_to_string(price, payload);

src/lib_binance_execution/binance_execution/usds_futures/messages/position_info_request.cpp

@@ -0,0 +1,16 @@
+#include "position_info_request.h"
+#include "binance_execution/type_converters.h"
+#include "lib/core/TimeStamp.h"
+
+namespace binance_execution::usds_futures {
+
+void PositionInfoRequest::init() {
+   timestamp = convert_time_stamp(bts::core::TimeStamp::now());
+}
+
+void PositionInfoRequest::init(std::string symbol) {
+   this->symbol = std::move(symbol);
+   timestamp = convert_time_stamp(bts::core::TimeStamp::now());
+}
+
+} // namespace binance_execution::usds_futures

src/lib_binance_execution/binance_execution/usds_futures/messages/position_info_request.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/json_formatter.h"
+
 #include <cstdint>
 #include <nlohmann/json.hpp>
 #include <string>
@@ -10,9 +13,17 @@ namespace binance_execution::usds_futures {
  * @a https://developers.binance.com/docs/derivatives/usds-margined-futures/user-data-streams/Event-Balance-and-Position-Update
  */
 struct PositionInfoRequest {
+   static constexpr const char *WS_METHOD = "v2/account.position";
+
+   std::string apiKey;
    std::string symbol;
    std::int64_t recvWindow;
    std::int64_t timestamp;
+   std::string signature;
+
+   void init();
+   void init(std::string symbol);
+   void to_signature_params(std::string &payload) const;
 };
 
 void to_json(nlohmann::json &j, const PositionInfoRequest &req);
@@ -24,14 +35,37 @@ void to_query_params(std::string &qps, const PositionInfoRequest &req);
 // Definition
 //////////////////////////////////////////////////
 
+inline void PositionInfoRequest::to_signature_params(
+    std::string &payload) const {
+   payload.clear();
+   payload.append("apiKey=");
+   payload.append(apiKey);
+   if (recvWindow > 0) {
+      payload.append("&recvWindow=");
+      payload.append(std::to_string(recvWindow));
+   }
+   if (!symbol.empty()) {
+      payload.append("&symbol=");
+      payload.append(symbol);
+   }
+   payload.append("&timestamp=");
+   payload.append(std::to_string(timestamp));
+}
+
 inline void to_json(nlohmann::json &j, const PositionInfoRequest &req) {
    j["timestamp"] = req.timestamp;
    if (!req.symbol.empty()) {
       j["symbol"] = req.symbol;
    }
+   if (!req.apiKey.empty()) {
+      j["apiKey"] = req.apiKey;
+   }
    if (req.recvWindow) {
       j["recvWindow"] = req.recvWindow;
    }
+   if (!req.signature.empty()) {
+      j["signature"] = req.signature;
+   }
 }
 
 inline void from_json(const nlohmann::json &j, PositionInfoRequest &req) {
@@ -40,13 +74,25 @@ inline void from_json(const nlohmann::json &j, PositionInfoRequest &req) {
    if (it != j.end()) {
       it->get_to(req.symbol);
    }
+   it = j.find("apiKey");
+   if (it != j.end()) {
+      it->get_to(req.apiKey);
+   }
    it = j.find("recvWindow");
    if (it != j.end()) {
       it->get_to(req.recvWindow);
    }
+   it = j.find("signature");
+   if (it != j.end()) {
+      it->get_to(req.signature);
+   }
 }
 
 inline void to_query_params(std::string &qps, const PositionInfoRequest &req) {
+   if (!req.apiKey.empty()) {
+      qps.append("apiKey=");
+      qps.append(req.apiKey);
+   }
    qps.append("timestamp=");
    qps.append(std::to_string(req.timestamp));
    if (!req.symbol.empty()) {
@@ -60,3 +106,9 @@ inline void to_query_params(std::string &qps, const PositionInfoRequest &req) {
 }
 
 } // namespace binance_execution::usds_futures
+
+FMT_IMPL(
+    binance_execution::usds_futures::PositionInfoRequest,
+    r,
+    "{}",
+    bts::io::json_formatter(r));

src/lib_binance_execution/binance_execution/usds_futures/usds_futures_gateway.cpp

@@ -1,23 +1,33 @@
 #include "usds_futures_gateway.h"
 #include "binance_execution/enums/order_status.h"
+#include "binance_execution/enums/time_in_force.h"
 #include "binance_execution/execution_report_store.h"
 #include "binance_execution/gateway_config.h"
 #include "binance_execution/listen_key_requestor.h"
+#include "binance_execution/position_calculator.h"
 #include "binance_execution/simple_order_key_map.h"
+#include "binance_execution/type_converters.h"
 #include "binance_execution/type_validators.h"
 #include "binance_execution/usds_futures/messages/create_order_request.h"
 #include "binance_execution/usds_futures/messages/modify_order_request.h"
+#include "binance_execution/usds_futures/messages/position_info.h"
+#include "binance_execution/usds_futures/messages/position_info_request.h"
 #include "binance_execution/usds_futures/rest_api_client.h"
 #include "binance_execution/usds_futures/user_data_stream/user_data_subscription.h"
 #include "binance_execution/websocket_api_request.h"
+#include "binance_execution/websocket_api_response.h"
 #include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/api/services/AdminService.pb.h"
 #include "lib/api/services/OrderService.h"
 #include "lib/collections/concurrent_queue.h"
+#include "lib/collections/concurrent_unordered_map.h"
 #include "lib/config/protos/jetfix.pb.h"
+#include "lib/core/TimeStamp.h"
+#include "lib/core/start_stop_state_machine.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/Side.h"
+#include "lib/io/JsonReader.h"
 #include "lib/io/json_formatter.h"
 #include "lib/io/websocket/websocket_client.h"
 #include "lib/logging/logging.h"
@@ -39,6 +49,7 @@
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/utils/Uuid.h"
 #include "lib/utils/bts_lock_guard.h"
+#include "lib/utils/observable_value_store.h"
 #include "lib/utils/overloaded.h"
 #include "lib/utils/thread_manager.h"
 
@@ -74,6 +85,8 @@ namespace binance_execution::usds_futures {
    LOG_TRACE_PARAMS(                                                           \
        OrderMan, level, std::string("binance_ufuts_gw"), std::string(name),    \
        params)
+#define DIAG_TRACE_PARAMS(name, params)                                        \
+   _LOG_TRACE_PARAMS(bts::Diag, name, params)
 #define ERROR_TRACE_PARAMS(name, params)                                       \
    _LOG_TRACE_PARAMS(bts::Error, name, params)
 
@@ -87,10 +100,14 @@ namespace binance_execution::usds_futures {
 using binance_execution::usds_futures::user_data_stream::AccountUpdate;
 using binance_execution::usds_futures::user_data_stream::OrderUpdate;
 using bts::collections::ConcurrentQueue;
+using bts::collections::ConcurrentUnorderedMap;
 using bts::core::Delegate;
+using bts::core::StartStopStateMachine;
+using bts::core::StartStopStateMachineBase;
 using bts::core::TimeSpan;
 using bts::core::TimeStamp;
 using bts::io::json_formatter;
+using bts::io::JsonReader;
 using bts::orderman::CancelOrderMsg;
 using bts::orderman::CancelReplaceOrderMsg;
 using bts::orderman::CreateOrderMsg;
@@ -101,6 +118,7 @@ using bts::orderman::OrderKey;
 using bts::orderman::OrderMsg;
 using bts::orderman::RFQMsg;
 using bts::threading::configure_current_thread;
+using bts::utils::FunctionalObservableValueStore;
 using bts::websocket::WebsocketClient;
 using bts::websocket::WebsocketClientConfig;
 using nlohmann::json;
@@ -141,54 +159,91 @@ struct UsdsFuturesGateway::PImpl {
              [this](const auto &req) {
                 return rest_api_client.refresh_listen_key(req);
              }),
-         orders(this->config.account, this->jetfix_connection) {
+         orders(this->config.account, this->jetfix_connection),
+         is_ready_store(
+             [&]() {
+                BTS_LOCK_GUARD(sm.mutex());
+                const bool ws_ready = ws_client && ws_client->is_connected();
+                const bool ud_ready =
+                    user_data_sub && user_data_sub->is_ready();
+                return sm.state() == StartStopStateMachineBase::RUNNING &&
+                       ws_ready && ud_ready;
+             },
+             false) {
       bts_assert(this->jetfix_connection->venue() == VENUE);
       bts_assert(!this->jetfix_connection->name().empty());
       pos.exchange = VENUE;
+      sm.set_name("binance_ufuts_gw");
+      sm.set_subsystem(bts::log::OrderMan);
+      sm.start_core_impl.bind<&PImpl::start_impl>(*this);
+      sm.stop_core_impl.bind<&PImpl::stop_impl>(*this);
+      is_ready_store.value_changed().bind<&PImpl::on_is_ready_changed>(*this);
    }
 
+   ~PImpl() { stop(); }
+
+   enum RequestType {
+      CREATE_ORDER,
+      MODIFY_ORDER,
+      CANCEL_ORDER,
+      POSITION,
+   };
+
    bts::md::SecurityOracle sec_oracle;
    GatewayConfig config;
    std::shared_ptr<bts::config::JetFIX_Connection> jetfix_connection;
    optional<WebsocketClient> ws_client;
-   optional<thread> ws_run_thread;
+   optional<thread> pos_snapshot_thread;
+   std::atomic<bool> is_ready_dirty = false;
 
    RestApiClient rest_api_client;
    optional<user_data_stream::UserDataSubscription> user_data_sub;
    Delegate<void(user_data_stream::AccountUpdate)> account_update_callback;
    Delegate<void(user_data_stream::OrderUpdate)> order_update_callback;
    ListenKeyRequestor listen_key_requestor;
+   StartStopStateMachine sm;
    SimpleOrderKeyMap key_map;
 
-   std::mutex instance_mx;
+   ConcurrentUnorderedMap<string, RequestType> pending_reqs;
+   ConcurrentUnorderedMap<string, CreateOrderMsg> pending_create_order_reqs;
    ConcurrentQueue<AccountUpdate> incoming_account_updates;
+   vector<PositionInfo> incoming_position_snapshot;
    ConcurrentQueue<OrderUpdate> incoming_order_updates;
    ConcurrentQueue<shared_ptr<const ExecutionReport>> internal_order_rejects;
    vector<AccountUpdate> account_updates_vec;
 
    ExecutionReportStore orders;
+   FunctionalObservableValueStore<bool> is_ready_store;
    ExecutionGatewayCallbacks callbacks;
+   optional<PositionCalculator> pos_calc;
    bts::orderman::ExchangePosition pos;
    bts::api::admin::BTSAccount account;
 
    template <WebsocketApiRequestParams T>
-   WebsocketApiRequest<T> build_request(T &&params) {
+   WebsocketApiRequest<T> build_request(T &&params, RequestType type) {
       params.apiKey = config.api_key;
       WebsocketApiRequest<T> req;
       req.params = std::move(params);
       req.id = bts::utils::uuid();
       req.sign(config.secret_key);
+      bts_assert(pending_reqs.try_emplace(req.id, type).second);
       return req;
    }
 
-   bool is_ready() {
-      BTS_LOCK_GUARD(instance_mx);
-      return ws_client && ws_client->is_connected() && user_data_sub &&
-             user_data_sub->is_ready();
+   void on_is_ready_changed(bool value) {
+      INFO_TRACE_PARAMS("is_ready_changed", json_formatter(value));
+      is_ready_dirty = true;
    }
 
    void start() {
       INFO_TRACE("start");
+      sm.start();
+      is_ready_store.refresh();
+   }
+
+   // should not be called directly (use the state machine)
+   void start_impl() {
+      INFO_TRACE("start_impl");
       listen_key_requestor.callback().bind<&PImpl::on_listen_key_>(*this);
       listen_key_requestor.start();
       // Init websocket
@@ -202,6 +257,21 @@ struct UsdsFuturesGateway::PImpl {
       ws_cli.close_handler().bind<&PImpl::on_close>(*this);
       ws_cli.connect();
       ws_cli.run_in_background();
+      pos_snapshot_thread.emplace([&]() {
+         INFO_TRACE_PARAMS("pos_snapshot", json_formatter("begin"));
+         while (!sm.is_stopping()) {
+            // random interval between 5 and 10 secs
+            auto req_time = TimeStamp::now() +
+                            TimeSpan::from_seconds(5 + TimeStamp::now_ns() % 5);
+            do {
+               // Only sleep for 1 second so we don't hang too long if stop is
+               // called, as it joins this thread.
+               std::this_thread::sleep_for(std::chrono::seconds(1));
+            } while (TimeStamp::now() < req_time && !sm.is_stopping());
+            get_position_info();
+         }
+         INFO_TRACE_PARAMS("pos_snapshot", json_formatter("end"));
+      });
    }
 
    void on_open(const std::error_code &ec) {
@@ -212,6 +282,7 @@ struct UsdsFuturesGateway::PImpl {
       }
       bts_assert_msgf(!ec, "Websocket opened with error. {}", ec.message());
       ws_client->ping_continuously(25);
+      is_ready_store.refresh();
    }
 
    void on_close(const std::error_code &ec) {
@@ -220,49 +291,90 @@ struct UsdsFuturesGateway::PImpl {
       } else {
          WARN_TRACE("on_close");
       }
+      is_ready_store.refresh();
    }
 
    void stop() {
       INFO_TRACE("stop");
+      sm.stop();
+      is_ready_store.refresh();
+   }
+
+   void stop_impl() {
+      INFO_TRACE("stop_impl");
       user_data_sub.reset();
       ws_client->disconnect();
       ws_client.reset();
+      if (pos_snapshot_thread) {
+         pos_snapshot_thread->join();
+         pos_snapshot_thread.reset();
+      }
    }
 
    void on_account_update(user_data_stream::AccountUpdate u) {
       INFO(R"([binance_ufuts_account_upd.json]{})", json_formatter(u));
-      incoming_account_updates.push(std::move(u));
+      sm.invoke_if_active(
+          [&](auto) { incoming_account_updates.push(std::move(u)); });
    }
 
    void on_order_update(user_data_stream::OrderUpdate u) {
       INFO(R"([binance_ufuts_order_upd.json]{})", json_formatter(u));
-      incoming_order_updates.push(std::move(u));
+      sm.invoke_if_active(
+          [&](auto) { incoming_order_updates.push(std::move(u)); });
    }
 
+   void on_user_data_is_ready(bool) { is_ready_store.refresh(); }
+
    void on_listen_key_(const string &listen_key) {
       INFO_TRACE_PARAMS("on_listen_key", listen_key);
-      user_data_sub.reset();
-      if (listen_key.empty()) {
-         LOG(OrderMan, bts::Error,
-             "Unable to subscribe to user data because 'listen_key' "
-             "is empty.");
-         return;
-      }
-      auto &uds = user_data_sub.emplace(
-          config.ws_stream_endpoint + "/ws/" + listen_key);
-      uds.account_update_callback().bind<&PImpl::on_account_update>(*this);
-      uds.order_update_callback().bind<&PImpl::on_order_update>(*this);
-      uds.start();
+      sm.invoke_if_active([&](auto) {
+         user_data_sub.reset();
+         if (listen_key.empty()) {
+            LOG(OrderMan, bts::Error,
+                "Unable to subscribe to user data because 'listen_key' "
+                "is empty.");
+            return;
+         }
+         auto &uds = user_data_sub.emplace(
+             config.ws_stream_endpoint + "/ws/" + listen_key);
+         uds.account_update_callback().bind<&PImpl::on_account_update>(*this);
+         uds.order_update_callback().bind<&PImpl::on_order_update>(*this);
+         uds.is_ready_callback().bind<&PImpl::on_user_data_is_ready>(*this);
+         uds.start();
+      });
    }
 
-   void on_order_ack(const json &msg) {
-      BTS_LOCK_GUARD(instance_mx);
-      INFO(R"([binance_ufuts_order_ack.json]{})", msg.dump());
-      auto id_it = msg.find("id");
-      bts_assert_msgf(
-          id_it != msg.end(), "Received order ack without id. {}", msg.dump());
-      string id;
-      id_it->get_to(id);
+   void on_order_ack(
+       RequestType req_type, const WebsocketApiResponse<nlohmann::json> &res) {
+      INFO(R"([binance_ufuts_order_ack.json]{})", bts::io::json_formatter(res));
+      // We generate exe reports from the order update stream (not the ack), with
+      // a caveat for new-order rejects because they aren't visible in the stream.
+      // Presumably, they're never actually entered into binance's order sys;
+      // they're "rejected" before that happens.
+      if (req_type == CREATE_ORDER && !res.error.is_null()) {
+         const CreateOrderMsg &bts_req = pending_create_order_reqs[res.id];
+         shared_ptr<const ExecutionReport> er_p = orders.reject(
+             bts_req,
+             fmt::format("Rejected by exchange. {}", res.error.dump()));
+         const auto &er = *er_p;
+         orders.erase(er.order_key);
+         key_map.remove(er.order_key);
+         internal_order_rejects.push(std::move(er_p));
+      }
+      pending_create_order_reqs.erase(res.id);
+   }
+
+   void on_position_snapshot(const json &msg) {
+      INFO(R"([binance_ufuts_pos_snap.json]{})", msg.dump());
+      BTS_LOCK_GUARD(sm.mutex());
+      incoming_position_snapshot.clear();
+      try {
+         msg.at("result").get_to(incoming_position_snapshot);
+      } catch (const std::exception &ex) {
+         ERROR(
+             R"([binance_ufuts_pos_snap_err.json]{ "err": "{}" })",
+             string{ex.what()});
+      }
    }
 
    void on_message(const string &msg_str) {
@@ -272,28 +384,47 @@ struct UsdsFuturesGateway::PImpl {
       DIAG_FN("on_message", "{}", msg_str);
       // Convert to nlohmann::json
       json msg;
+      WebsocketApiResponse<nlohmann::json> res;
       try {
          msg = json::parse(msg_str);
+         msg.get_to(res);
       } catch (const exception &ex) {
          ERROR_FN(
              "on_message", "Failed to parse response json: {}.\n{}",
              string{ex.what()}, msg_str);
          return;
       }
-      auto id_it = msg.find("id");
-      if (id_it != msg.end()) {
-         try {
-            on_order_ack(msg);
-         } catch (const exception &ex) {
-            ERROR_FN(
-                "on_message", "message handler threw. msg: {}; ex: {}", msg_str,
-                string{ex.what()});
+      try {
+         auto pr_it = pending_reqs.find(res.id);
+         if (pr_it == pending_reqs.end()) {
+            ERROR(
+                R"([binance_ufuts_unsolicited_res.json]{{ "id": "{}", "ts": {} }})",
+                res.id, bts::core::TimeStamp::now_ns());
+            return;
          }
+         const RequestType req_type = pr_it->second;
+         pending_reqs.erase(pr_it);
+         switch (req_type) {
+         case CREATE_ORDER:
+         case MODIFY_ORDER:
+         case CANCEL_ORDER:
+            on_order_ack(req_type, res);
+            break;
+         case POSITION:
+            on_position_snapshot(msg);
+            break;
+         default:
+            break;
+         }
+      } catch (const exception &ex) {
+         ERROR_FN(
+             "on_message", "message handler threw. msg: {}; ex: {}", msg_str,
+             string{ex.what()});
       }
    }
 
    void create_order(const CreateOrderMsg &m) {
-      if (!is_ready()) {
+      if (!is_ready_store.value()) {
          LOG_CREATE_ORDER(Error, m, "REJECTED_BEFORE_SEND");
          internal_order_rejects.push(orders.reject(m, "Gateway not ready"));
          return;
@@ -308,7 +439,9 @@ struct UsdsFuturesGateway::PImpl {
          return;
       }
       LOG_CREATE_ORDER(Info, m, "ABOUT_TO_SEND");
-      const auto req = build_request(std::move(params));
+      const auto req =
+          build_request(std::move(params), RequestType::CREATE_ORDER);
+      pending_create_order_reqs[req.id] = m;
       const auto ec = ws_client->send_text(req.to_string());
       if (ec) {
          LOG_CREATE_ORDER(Error, m, "WS_ERROR", ec.message());
@@ -330,7 +463,8 @@ struct UsdsFuturesGateway::PImpl {
          return;
       }
       LOG_MODIFY_ORDER(Info, m, "ABOUT_TO_SEND");
-      const auto req = build_request(std::move(params));
+      const auto req =
+          build_request(std::move(params), RequestType::MODIFY_ORDER);
       const auto ec = ws_client->send_text(req.to_string());
       if (ec) {
          LOG_MODIFY_ORDER(Error, m, "WS_ERROR", ec.message());
@@ -351,14 +485,44 @@ struct UsdsFuturesGateway::PImpl {
          return;
       }
       LOG_CANCEL_ORDER(Info, m, "ABOUT_TO_SEND");
-      const auto req = build_request(std::move(params));
+      const auto req =
+          build_request(std::move(params), RequestType::CANCEL_ORDER);
       const auto ec = ws_client->send_text(req.to_string());
       if (ec) {
          LOG_CANCEL_ORDER(Error, m, "WS_ERROR", ec.message());
       }
    }
 
+   void get_position_info() {
+      sm.invoke_if_active([&](auto) {
+         if (!is_ready_store.value()) {
+            ERROR_FN("get_position_info", "Cannot get. Not ready.");
+            return;
+         }
+         PositionInfoRequest params;
+         params.init();
+         INFO_FN(
+             "get_position_info", "[binance_gw_pos_req.json]{}",
+             json_formatter(params));
+         const auto req =
+             build_request(std::move(params), RequestType::POSITION);
+         const auto ec = ws_client->send_text(req.to_string());
+         if (ec) {
+            ERROR_FN("get_position_info", "{}", ec.message());
+         }
+      });
+   }
+
+   void dispatch_is_ready() {
+      BTS_LOCK_GUARD(sm.mutex());
+      if (is_ready_dirty) {
+         is_ready_dirty = false;
+         callbacks.gw_status_change(is_ready_store.value());
+      }
+   }
+
    void dispatch_internal_rejects() {
+      BTS_LOCK_GUARD(sm.mutex());
       shared_ptr<const ExecutionReport> curr;
       while (internal_order_rejects.pop(curr)) {
          INFO_FN("dispatch_internal_rejects", "{}", curr->to_string());
@@ -441,50 +605,38 @@ struct UsdsFuturesGateway::PImpl {
       while (incoming_account_updates.pop(account_updates_vec.emplace_back())) {
       }
       account_updates_vec.pop_back();
-      if (account_updates_vec.empty()) {
-         return;
-      }
-      auto &dest = pos.total_exchange_positions;
-      for (const auto &acct_upd : account_updates_vec) {
-         for (const auto &pos : acct_upd.data.positions) {
-            auto it = std::lower_bound(
-                dest.begin(), dest.end(), pos.symbol,
-                [](const bts::orderman::ExchangePosition::UnitPosition &u,
-                   const string &ticker) {
-                   return u.exchange_sec_desc < ticker;
-                });
-            if (it == dest.end() || it->exchange_sec_desc != pos.symbol) {
-               const auto *sec = sec_oracle.get_security(
-                   VENUE, bts::api::SECURITYTYPE_FUTURE, pos.symbol);
-               if (sec == nullptr) {
-                  continue;
-               }
-               bts::orderman::ExchangePosition::UnitPosition new_pos;
-               new_pos.exchange_sec_desc = pos.symbol;
-               new_pos.security = sec;
-               new_pos.account_id = account.id();
-               new_pos.account_name = account.name();
-               it = dest.insert(it, new_pos);
-            }
-            switch (pos.position_side.value) {
-            case PositionSide::BOTH:
-               it->position = pos.position_amount;
-               break;
-            case PositionSide::SHORT:
-               it->position = -pos.position_amount;
-               break;
-            case PositionSide::LONG:
-               it->position = pos.position_amount;
-               break;
-            default:
-               bts_assert_msgf(
-                   false, "Invalid PositionSide '{}'", pos.position_side);
-               break;
-            }
-            it->avg_filled_price.value() = pos.break_even_price;
+      bool is_dirty = false;
+      {
+         BTS_LOCK_GUARD(sm.mutex());
+         if (!pos_calc) {
+            DIAG_TRACE_PARAMS(
+                "dispatch_account_updates",
+                json_formatter(
+                    "PositionCalculator not ready (waiting for admin data)"));
+            // We drain the queue even if pos calc isn't ready to prevent buffer
+            // overflow. This is ok because we take periodic position snapshots
+            // so the full state is always recovered.
+            return;
          }
+         for (const auto &acct_upd : account_updates_vec) {
+            for (const auto &pu : acct_upd.data.positions) {
+               INFO(R"([binance_ufuts_pos.json]{})", json_formatter(pu));
+               is_dirty |= pos_calc->on_update(
+                   pu.symbol, pu.position_side, pu.position_amount,
+                   pu.break_even_price, acct_upd.transaction_time, &pos);
+            }
+         }
+         for (const auto &ps : incoming_position_snapshot) {
+            INFO(R"([binance_ufuts_pos_snap.json]{})", json_formatter(ps));
+            is_dirty |= pos_calc->on_update(
+                ps.symbol, ps.positionSide, ps.positionAmt, ps.breakEvenPrice,
+                ps.updateTime, &pos);
+         }
+         incoming_position_snapshot.clear();
+      }
+      if (is_dirty) {
+         callbacks.exchange_position_state(pos);
       }
-      callbacks.exchange_position_state(pos);
    }
 };
 
@@ -520,6 +672,7 @@ void UsdsFuturesGateway::on_admin_data(
    if (update.events == nullptr) {
       return;
    }
+   std::lock_guard<std::recursive_mutex> lock{me->sm.mutex()};
    const auto *account = bts::admin::AdminServiceUtils::find_account(
        *update.events->back().admin_data, me->config.account);
    if (account == nullptr) {
@@ -534,6 +687,10 @@ void UsdsFuturesGateway::on_admin_data(
       return;
    }
    me->account.CopyFrom(*account);
+   me->pos_calc.reset();
+   me->pos_calc.emplace(
+       me->account.name(), me->account.id(), VENUE,
+       bts::api::SECURITYTYPE_FUTURE, me->sec_oracle);
    INFO_FN("on_admin_data", "Resolved gateway account '{}'", account->name());
 }
 
@@ -561,6 +718,7 @@ void UsdsFuturesGateway::send_fix(
 }
 
 void UsdsFuturesGateway::dispatch_incoming_fix_messages() {
+   me->dispatch_is_ready();
    me->dispatch_internal_rejects();
    me->dispatch_order_updates();
    me->dispatch_account_updates();

src/lib_binance_execution/binance_execution/usds_futures/user_data_stream/user_data_subscription.cpp

@@ -42,6 +42,7 @@ struct UserDataSubscription::PImpl {
    optional<thread> ws_run_thread;
    optional<thread> ws_ping_thread;
 
+   Delegate<void(bool)> is_ready_callback;
    Delegate<void(AccountUpdate)> account_update_callback;
    Delegate<void(OrderUpdate)> order_update_callback;
 
@@ -57,6 +58,7 @@ struct UserDataSubscription::PImpl {
       ws_stream_client_cfg.name = "BinanceUsdsFutures_UserData";
       auto &ws_cli = ws_client.emplace(ws_stream_client_cfg);
       ws_cli.message_handler().bind<&PImpl::on_message>(*this);
+      ws_cli.open_handler().bind<&PImpl::on_open>(*this);
       ws_cli.close_handler().bind<&PImpl::on_close>(*this);
       INFO_FN("start", "Creating websocket client. url: {}", url);
       ws_cli.connect();
@@ -137,9 +139,17 @@ struct UserDataSubscription::PImpl {
       ERROR_FN("on_message", "Failed to process response json: {}", msg_str);
    }
 
+   void on_open(const std::error_code &ec) {
+      bts_assert_msgf(!ec, "Websocket opened with error. {}", ec.message());
+      if (!ec) {
+         is_ready_callback(true);
+      }
+   }
+
    void on_close(const std::error_code &ec) {
       bts_assert_msgf(
           false, "Websocket stream client disconnected. {}", ec.message());
+      is_ready_callback(false);
    }
 };
 
@@ -162,6 +172,10 @@ void UserDataSubscription::stop() {
    me->stop();
 }
 
+Delegate<void(bool)> &UserDataSubscription::is_ready_callback() {
+   return me->is_ready_callback;
+}
+
 Delegate<void(AccountUpdate)> &UserDataSubscription::account_update_callback() {
    return me->account_update_callback;
 }

src/lib_binance_execution/binance_execution/usds_futures/user_data_stream/user_data_subscription.h

@@ -8,7 +8,6 @@
 
 namespace binance_execution::usds_futures::user_data_stream {
 
-
 /**
  * @a https://developers.binance.com/docs/derivatives/usds-margined-futures/user-data-streams
  */
@@ -20,6 +19,7 @@ class UserDataSubscription {
    void start();
    bool is_ready() const;
    void stop();
+   bts::core::Delegate<void(bool)> &is_ready_callback();
    bts::core::Delegate<void(AccountUpdate)> &account_update_callback();
    bts::core::Delegate<void(OrderUpdate)> &order_update_callback();
 

src/lib_binance_execution/binance_execution/websocket_api_request.cpp

@@ -5,6 +5,7 @@
 #include "binance_execution/usds_futures/messages/cancel_order_request.h"
 #include "binance_execution/usds_futures/messages/create_order_request.h"
 #include "binance_execution/usds_futures/messages/modify_order_request.h"
+#include "binance_execution/usds_futures/messages/position_info_request.h"
 #include "lib/core/TimeStamp.h"
 #include "lib/logging/logging.h"
 #include "lib/utils/crypto/hmac.h"
@@ -59,5 +60,6 @@ REQUEST_DEF(spots::CancelOrderRequest);
 REQUEST_DEF(usds_futures::CreateOrderRequest);
 REQUEST_DEF(usds_futures::ModifyOrderRequest);
 REQUEST_DEF(usds_futures::CancelOrderRequest);
+REQUEST_DEF(usds_futures::PositionInfoRequest);
 
 } // namespace binance_execution

src/lib_binance_execution/binance_execution/websocket_api_response.cpp

@@ -0,0 +1,44 @@
+#include "websocket_api_response.h"
+
+namespace binance_execution {
+
+template <typename Result>
+std::string WebsocketApiResponse<Result>::to_string() const {
+   std::string str = fmt::format("{}", *this);
+   return str;
+}
+
+template <typename Result>
+void to_json(nlohmann::json &j, const WebsocketApiResponse<Result> &req) {
+   j["id"] = req.id;
+   j["status"] = req.status;
+   j["result"] = req.result;
+   if (!req.error.is_null() || req.status >= 400) {
+      j["error"] = req.error;
+   }
+}
+
+template <typename Result>
+void from_json(const nlohmann::json &j, WebsocketApiResponse<Result> &req) {
+   j.at("id").get_to(req.id);
+   j.at("status").get_to(req.status);
+   auto it = j.find("result");
+   if (it != j.end()) {
+      it->get_to(req.result);
+   }
+   it = j.find("error");
+   if (it != j.end()) {
+      it->get_to(req.error);
+   }
+}
+
+#define RESPONSE_DEF(T)                                                        \
+   template struct WebsocketApiResponse<T>;                                    \
+   template void to_json(                                                      \
+       nlohmann::json &j, const WebsocketApiResponse<T> &req);                 \
+   template void from_json(                                                    \
+       const nlohmann::json &j, WebsocketApiResponse<T> &req)
+
+RESPONSE_DEF(nlohmann::json);
+
+} // namespace binance_execution

src/lib_binance_execution/binance_execution/websocket_api_response.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/json_formatter.h"
+
+#include <nlohmann/json.hpp>
+
+namespace binance_execution {
+
+template <typename Result>
+struct WebsocketApiResponse {
+   std::string id;
+   std::int64_t status;
+   Result result;
+   nlohmann::json error;
+
+   std::string to_string() const;
+};
+
+template <typename Result>
+void to_json(nlohmann::json &j, const WebsocketApiResponse<Result> &req);
+
+template <typename Result>
+void from_json(const nlohmann::json &j, WebsocketApiResponse<Result> &req);
+
+} // namespace binance_execution
+
+template <typename Result>
+FMT_IMPL_T(
+    binance_execution::WebsocketApiResponse<Result>,
+    r,
+    R"({{"id": "{}", "status": {}, "result": {} }})",
+    r.id,
+    r.status,
+    bts::io::json_formatter(r.result));

src/server/sparkd/configurations/alphaworx_prod/spark.conf

@@ -112,7 +112,7 @@ OkxExeCfg {
     secret_key: "240038D7A28D7F9461247CFCB8E43710"
     api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
     passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-    ws_url: "wss://wsaws.okx.com:8443"
+    ws_url: "wss://ws.okx.com:8443"
     rest_url: "https://www.okx.com"
     ws_ping_timeout_seconds: 10 
     ws_ping_interval_seconds: 25

src/server/sparkd/configurations/alphaworx_zen_hk_live/spark.conf

@@ -112,7 +112,7 @@ OkxExeCfg {
     secret_key: "240038D7A28D7F9461247CFCB8E43710"
     api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
     passphrase: ":,7i%tA%ivfj%SCbX?Ut"
-    ws_url: "wss://wsaws.okx.com:8443"
+    ws_url: "wss://ws.okx.com:8443"
     rest_url: "https://www.okx.com"
     ws_ping_timeout_seconds: 10 
     ws_ping_interval_seconds: 10

src/server/sparkd/configurations/binance_dev/spark.conf

@@ -43,10 +43,6 @@ Logging {
     }
     file {
       default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
       subsystem {
         name: "OrderMan"
         level: "Diag"

tools/binance/binance.env

@@ -1,10 +1,15 @@
 BINANCE_USDS_FUTURES_PROD_URL="https://fapi.binance.com"
+BINANCE_USDS_FUTURES_PROD_WS_URL="wss://fstream.binance.com:443/ws"
+
 BINANCE_USDS_FUTURES_DEV_URL="https://testnet.binancefuture.com"
+BINANCE_USDS_FUTURES_DEV_WS_URL="wss://stream.binancefuture.com:443/ws"
 BINANCE_USDS_FUTURES_DEV_API_KEY="b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0"
 BINANCE_USDS_FUTURES_DEV_SECRET_KEY="a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492"
 
 BINANCE_USDS_FUTURES_EXCHANGE_INFO_PATH="/fapi/v1/exchangeInfo"
 
+BINANCE_USDS_FUTURES_USER_DATA_PATH="/ws/${LISTEN_KEY}"
+
 BINANCE_USDS_FUTURES_DEPTH_PATH="/fapi/v1/depth"
 BINANCE_USDS_FUTURES_DEPTH_PARAMS='symbol=${SYMBOL}&limit=${LIMIT:-5}'
 
@@ -60,7 +65,7 @@ BINANCE_USDS_FUTURES_QUERY_ALL_ORDERS_METHOD="GET"
 BINANCE_USDS_FUTURES_QUERY_ALL_ORDERS_IS_SIGNED=1
 
 BINANCE_USDS_FUTURES_OPEN_ORDERS_PATH="/fapi/v1/openOrders"
-BINANCE_USDS_FUTURES_OPEN_ORDERS_PARAMS='symbol=${SYMBOL}&recvWindow=${RECV_WINDOW:-6000}'
+BINANCE_USDS_FUTURES_OPEN_ORDERS_PARAMS='symbol=${SYMBOL:-OMIT}&recvWindow=${RECV_WINDOW:-6000}'
 BINANCE_USDS_FUTURES_OPEN_ORDERS_METHOD="GET"
 BINANCE_USDS_FUTURES_OPEN_ORDERS_IS_SIGNED=1
 
@@ -97,15 +102,19 @@ BINANCE_COIN_FUTURES_OPEN_INTEREST_PATH="/dapi/v1/openInterest"
 BINANCE_COIN_FUTURES_LONG_SHORT_RATIO_PATH="/futures/data/globalLongShortAccountRatio"
 
 BINANCE_PROD_URL="https://api.binance.com"
+BINANCE_PROD_WS_URL="wss://stream.binance.com:9443/ws"
 BINANCE_DEV_URL="https://testnet.binance.vision"
+BINANCE_DEV_WS_URL="wss://stream.testnet.binance.vision:9443/ws"
 BINANCE_DEV_API_KEY="QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd"
 BINANCE_DEV_SECRET_KEY="7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T"
 
+BINANCE_EXCHANGE_INFO_PATH="/api/v3/exchangeInfo"
+
 BINANCE_DEPTH_PATH="/api/v3/depth"
 BINANCE_DEPTH_PARAMS='symbol=${SYMBOL}&limit=${LIMIT:-5}'
 
 BINANCE_OPEN_ORDERS_PATH="/api/v3/openOrders"
-BINANCE_OPEN_ORDERS_PARAMS='symbol=${SYMBOL}'
+BINANCE_OPEN_ORDERS_PARAMS='symbol=${SYMBOL:-OMIT}'
 BINANCE_OPEN_ORDERS_IS_SIGNED=1
 
 BINANCE_ACCOUNT_PATH="/api/v3/account"

tools/binance/binance_client.py

@@ -0,0 +1,386 @@
+import argparse
+import hmac
+import hashlib
+import os
+import logging
+import sys
+import time
+import websocket
+import json
+import requests
+from string import Template
+from threading import Thread
+from typing import Any, Callable, Optional, TextIO
+
+from binance_env_helper import (
+    required_env_var,
+    expand_vars,
+    expand_defaults,
+    load_env,
+)
+
+
+def hmac_hashing(api_secret: str, payload: str):
+    m = hmac.new(api_secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256)
+    return m.hexdigest()
+
+
+def rest_request(base_url: str, api_key: str, api_secret: str, parameters: dict = {}):
+    parameters["timestamp"] = int(time.time() * 1000)
+    sorted_keys = list(parameters)
+    sorted_keys.sort()
+    query_params = "&".join([f"{k}={parameters[k]}" for k in sorted_keys])
+    signature = hmac_hashing(api_secret=api_secret, payload=query_params)
+    parameters["signature"] = signature
+    query_params += "&signature=" + signature
+    url = base_url + "/api/v3/userDataStream" + "?" + query_params
+    res = requests.post(url, headers={"X-MBX-APIKEY": api_key})
+    return res.text
+
+
+class CircularBufferWriter:
+    def __init__(self, buffer_line_count: int, sink: TextIO = sys.stdout):
+        self.buffer_lines = []
+        self.buffer_line_count = buffer_line_count
+        self.sink = sink
+
+    def print(self, msg):
+        if self.buffer_line_count <= 0:
+            self.sink.write(msg + "\n")
+            return
+        for _ in range(0, len(self.buffer_lines)):
+            self.sink.write("\x1b[1A\x1b[2K")  # move up cursor and delete whole line
+        if len(self.buffer_lines) == self.buffer_line_count:
+            self.buffer_lines.pop()
+        self.buffer_lines.insert(0, msg)
+        lines = ""
+        num_lines = len(self.buffer_lines)
+        for i in range(0, num_lines):
+            lines += self.buffer_lines[num_lines - i - 1] + "\n"
+        self.sink.write(lines)
+
+
+def _get_arg_parser():
+    # Cli Args
+    arg_parser = argparse.ArgumentParser(description="Request market data at Binance")
+    arg_parser.add_argument(
+        "--streams",
+        help="Streams to subscribe (e.g, btcusdt@depth5@500ms)",
+    )
+    arg_parser.add_argument(
+        "--prod", action="store_const", const="PROD", dest="mode", help="Run at prod"
+    )
+    arg_parser.add_argument(
+        "--dev", action="store_const", const="DEV", dest="mode", help="Run at testnet"
+    )
+    arg_parser.add_argument(
+        "--spot",
+        action="store_const",
+        const="BINANCE",
+        dest="venue",
+        help="Run at spots",
+    )
+    arg_parser.add_argument("--op", help="The op to run")
+    arg_parser.add_argument(
+        "--param", help="Parameter(s) formatted as key=val)", nargs="+"
+    )
+    arg_parser.add_argument("--create-order", action="store_true", help="Create order")
+    arg_parser.add_argument(
+        "--usds-future",
+        action="store_const",
+        const="BINANCE_USDS_FUTURES",
+        dest="venue",
+        help="Run at usds futures",
+    )
+    arg_parser.add_argument("--user-data", action="store_true", help="User data")
+    arg_parser.add_argument(
+        "--exch-info",
+        action="store_const",
+        const="EXCHANGE_INFO",
+        dest="op",
+        help="Get exchangeInfo (REST)",
+    )
+    arg_parser.add_argument(
+        "--list-symbols", action="store_true", help="Get symbols (REST)"
+    )
+    arg_parser.add_argument(
+        "--listen-key",
+        action="store_const",
+        const="LISTEN_KEY",
+        dest="op",
+        help="Get listen key (needed for streams)",
+    )
+    arg_parser.add_argument(
+        "--list-streams", action="store_true", help="List all streams"
+    )
+    arg_parser.add_argument(
+        "--display-lines",
+        help="Cap the number of lines to display from incoming messages (applies circular buffer to console output).",
+    )
+    arg_parser.add_argument("--url", help="The url to send requests to")
+    arg_parser.set_defaults(mode="DEV", venue="BINANCE", user_data=False, op=None)
+    return arg_parser
+
+
+def _parse_bool(s: Optional[str]):
+    match s:
+        case "1":
+            return True
+        case "true":
+            return True
+        case "on":
+            return True
+        case "y":
+            return True
+        case "yes":
+            return True
+        case _:
+            return False
+
+
+class BinanceClient:
+    def __init__(self, venue="BINANCE", mode="DEV", logger=None):
+        self.venue = venue
+        self.mode = mode
+        self.rest_url = required_env_var(f"{venue}_{mode}_URL")
+        self.ws_url = required_env_var(f"{venue}_{mode}_WS_URL")
+        self.logger = logger or logging.getLogger(__name__)
+        self.api_key = os.environ.get(f"{self.venue}_{self.mode}_API_KEY")
+        self.secret_key = os.environ.get(f"{self.venue}_{self.mode}_SECRET_KEY")
+
+    def _get_base_url(self, op, is_ws: bool = False, vars: Optional[dict] = None):
+        """Gets the base url for the request (includes host and path parts, but not query params)."""
+        url_var_suffix = "WS_URL" if is_ws else "URL"
+        url = os.environ.get(f"{self.venue}_{self.mode}_{url_var_suffix}") or ""
+        req_path = Template(os.environ.get(f"{self.venue}_{op}_PATH") or "").substitute(
+            os.environ
+        )
+        if vars:
+            req_path = Template(req_path).substitute(vars)
+        if req_path:
+            n = 1 if req_path.startswith("/") else 0
+            n += 1 if self.rest_url.endswith("/") else 0
+            match n:
+                case 0:
+                    url += "/" + req_path
+                case 1:
+                    url += req_path
+                case 2:
+                    url += req_path[1:]
+        return url
+
+    def _get_params_str(self, op, **kwargs):
+        params = os.environ.get(f"{self.venue}_{op}_PARAMS") or ""
+        params = expand_vars(params, kwargs)
+        params = expand_defaults(params)
+        return params
+
+    def _get_params_dict(self, op, **kwargs):
+        params = self._get_params_str(op=op, **kwargs)
+        if not params:
+            return {}
+        return {p[0]: p[1] for p in map(lambda x: x.split("="), params.split("&"))}
+
+    def request(self, op, **kwargs):
+        is_signed = _parse_bool(os.environ.get(f"{self.venue}_{op}_IS_SIGNED"))
+        is_keyed = _parse_bool(os.environ.get(f"{self.venue}_{op}_IS_KEYED"))
+        method = os.environ.get(f"{self.venue}_{op}_METHOD")
+        params = self._get_params_dict(op, **kwargs)
+        base_url = self._get_base_url(op, is_ws=False)
+        res = None
+        headers = {"Content-Type": "application/json"}
+        if is_keyed or is_signed:
+            if not self.api_key:
+                raise ValueError("Api key not defined")
+            headers["X-MBX-APIKEY"] = self.api_key
+        if is_signed:
+            if not self.secret_key:
+                raise ValueError("Secret key not defined")
+            params["timestamp"] = str(int(time.time() * 1000))
+            base_url = self._get_base_url(op)
+            sorted_keys = list(params)
+            sorted_keys.sort()
+            params_str = "&".join([f"{k}={params[k]}" for k in sorted_keys])
+            signature = hmac_hashing(api_secret=self.secret_key, payload=params_str)
+            params_str += "&signature=" + signature
+            url = base_url + "?" + params_str
+            if not method:
+                method = "POST"
+            del headers["Content-Type"]
+            res = requests.request(method, url, headers=headers)
+        else:
+            params_str = "&".join([f"{k}={v}" for k, v in params.items()])
+            url = base_url
+            if params_str:
+                url += "?" + params_str
+            if not method:
+                method = "GET"
+            res = requests.request(method, url, headers=headers)
+        if not res.ok:
+            self.logger.error("Request failed. op=%s, res=\n%s", op, str(res))
+        return res
+
+    def subscribe_user_data(
+        self,
+        on_message: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
+        on_error: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
+        on_close: Optional[Callable[[websocket.WebSocket, Any, Any], None]] = None,
+    ):
+        listen_key_res = self.request("LISTEN_KEY")
+        if not listen_key_res.ok:
+            raise ValueError("Failed to get listen key")
+        listen_key = listen_key_res.json().get("listenKey")
+        url = self._get_base_url(
+            "USER_DATA", is_ws=True, vars={"LISTEN_KEY": listen_key}
+        )
+        ws_app = websocket.WebSocketApp(
+            url=url,
+            on_message=on_message,
+            on_error=on_error,
+            on_close=on_close,
+        )
+        ws_app.run_forever(ping_interval=600, ping_timeout=180)
+
+    def subscribe_streams(
+        self,
+        streams,
+        on_message: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
+        on_error: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
+        on_close: Optional[Callable[[websocket.WebSocket, Any, Any], None]] = None,
+        **kwargs,
+    ):
+        def on_open(ws: websocket.WebSocket):
+            def run(*args):
+                try:
+                    req = {
+                        "method": "SUBSCRIBE",
+                        "params": streams,
+                        "id": 1,
+                    }
+                    req_str = json.dumps(req)
+                    self.logger.info("Send: %s", req_str)
+                    ws.send_text(req_str)
+                except Exception as ex:
+                    self.logger.error("Error: %s", ex)
+                    self.logger.info("Disconnecting websocket")
+                    ws.close()
+
+            self.logger.info("Websocket connected")
+            th = Thread(target=run)
+            th.start()
+            th.join()
+
+        url = self._get_base_url("STREAMS", is_ws=True, **kwargs)
+        ws_app = websocket.WebSocketApp(
+            url=url,
+            on_open=on_open,
+            on_message=on_message,
+            on_error=on_error,
+            on_close=on_close,
+        )
+        self.logger.info("StreamSubscription @ %s", url)
+        ws_app.run_forever(ping_interval=600, ping_timeout=180)
+
+
+def _main():
+    load_env()
+    arg_parser = _get_arg_parser()
+    args = arg_parser.parse_args()
+    if args.list_streams:
+        stream_descs = [
+            "{symbol}@aggTrade",
+            "{symbol}@markPrice",
+            "{symbol}@markPrice@{interval}                    (intervals: 1s)",
+            "!markPrice@arr",
+            "!markPrice@arr@1s",
+            "{symbol}@kline_{interval}                        (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)",
+            "{pair}_{contractType}@continuousKline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)"
+            "{symbol}@miniTicker",
+            "!ticker@arr",
+            "{symbol}@ticker",
+            "!miniTicker@arr",
+            "{symbol}@bookTicker",
+            "!bookTicker",
+            "{symbol}@forceOrder",
+            "!forceOrder@arr",
+            "{symbol}@depth{levels}                           (levels: 5, 10, 20)",
+            "{symbol}@depth{levels}@{interval}                (levels: 5, 10, 20)",
+            "                                                 (intervals: 100ms, 500ms)",
+            "{symbol}@depth",
+            "{symbol}@depth@{interval}                        (intervals: 100ms, 500ms)",
+            "{symbol}@compositeIndex",
+            "!contractInfo",
+            "!assetIndex@arr",
+            "{assetSymbol}@assetIndex",
+        ]
+        print("\n".join(stream_descs))
+        return
+    logger = logging.getLogger()
+    cli = BinanceClient(args.venue, args.mode, logger=logger)
+    params = {kv[0]: kv[1] for kv in map(lambda x: x.split("="), args.param or [])}
+    if args.op:
+        res = cli.request(args.op, **params)
+        if res.ok:
+            print(res.text)
+    elif args.list_symbols:
+        # Get symbols
+        res = cli.request("EXCHANGE_INFO")
+        if res.status_code < 200 or res.status_code > 300:
+            sys.stderr.write(
+                f"Request failed. code: {res.status_code}, reason: {res.reason}\n"
+            )
+            return 1
+        syms = {}
+        res_j = json.loads(res.text)
+        for sym in res_j["symbols"]:
+            syms[sym["symbol"]] = sym
+        sym_list = list(syms)
+        sym_list.sort()
+        for sym in sym_list:
+            print(sym)
+    elif args.user_data:
+        # Subscribe user data
+        writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
+
+        def on_message(ws, msg):
+            logger.debug("on_message: %s", msg)
+            writer.print(msg)
+
+        def on_error(ws, err):
+            logger.error("on_error: %s", err)
+
+        def on_close(ws, code, msg):
+            logger.info("on_close (%d): %s", code, msg)
+
+        cli.subscribe_user_data(on_message, on_error, on_close)
+    elif args.streams:
+        # Subscribe streams
+        writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
+
+        def on_message(ws, msg):
+            logger.debug("Recv: %s", msg)
+            writer.print(msg)
+
+        def on_error(ws, err):
+            logger.error("Error: %s", err)
+
+        def on_close(ws, code, msg):
+            logger.info("Closed (%d): %s", code, msg)
+
+        cli.subscribe_streams(
+            streams=args.streams if isinstance(args.streams, list) else [args.streams],
+            on_message=on_message,
+            on_error=on_error,
+            on_close=on_close,
+        )
+    else:
+        sys.stderr.write(
+            "No data requested (run with --streams <stream1> <stream2> ... or --exch-info or --list-symbols)"
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    code = _main()
+    exit(code)

tools/binance/binance_env_helper.py

@@ -0,0 +1,99 @@
+from typing import Optional
+import dotenv
+import os
+
+
+def required_env_var(key):
+    val = os.environ.get(key)
+    if val is None:
+        raise ValueError(f"{key} env var not defined")
+    return val
+
+
+def expand_env(s: str, vars=os.environ):
+    if not s:
+        return ""
+    beg = 0
+    while True:
+        beg = s.find("${", beg)
+        if beg == -1 or beg > len(s) - 4:
+            break
+        end = s.find("}", beg + 2)
+        if end == -1:
+            break
+        k = s[beg:end]
+        v = vars.get(k) or ""
+        s = s[:beg] + v + s[end + 1:]
+        beg += len(v) + 1
+    return s
+
+
+def expand_var(req_params: str, k: str, v: str):
+    ku = str(k).upper()
+    substitution = f"${{{ku}}}"
+    pos = req_params.find(substitution)
+    end_pos = -1
+    if pos == -1:
+        substitution = f"${{{ku}:"
+        pos = req_params.find(substitution)
+        if pos == -1:
+            return req_params
+        end_pos = req_params.find("}", pos + len(substitution))
+        if end_pos == -1:
+            return req_params
+        end_pos += 1
+    else:
+        end_pos = pos + len(substitution)
+    if end_pos == -1:
+        raise ValueError(f"Invalid request params template '{req_params}'")
+    req_params = req_params[:pos] + v + req_params[end_pos:]
+    return req_params
+
+
+def expand_vars(req_params: str, key_vals: dict):
+    for k, v in key_vals.items():
+        req_params = expand_var(req_params, k, v)
+    return req_params
+
+
+def expand_defaults(req_params: str):
+    i = 0
+    prefix = ":-"
+    while i < len(req_params):
+        start = req_params.find("${", i)
+        if start == -1:
+            break
+        stop = req_params.find("}", start)
+        if stop == -1:
+            raise ValueError(f"Invalid param str {req_params}")
+        prefix_pos = req_params.find(prefix, start, stop)
+        if prefix_pos < start:
+            req_params = req_params[:start] + req_params[stop + 1:]
+            i = start
+            continue
+        default_val = req_params[prefix_pos + len(prefix):stop]
+        if default_val == "OMIT":
+            i = req_params.rfind("&", 0, prefix_pos-1)
+            if i == -1:
+                i = 0
+            req_params = req_params[:i] + req_params[stop + 1:]
+        else:
+            req_params = req_params[:start] + default_val + req_params[stop + 1:]
+            i = start + len(default_val) + 1
+    if req_params.startswith("&"):
+        req_params = req_params[1:]
+    return req_params
+
+
+def load_env(env_path: Optional[str] = None):
+    if not env_path:
+        env_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                "binance.env")
+    dotenv.load_dotenv(env_path, interpolate=False)
+
+
+if __name__ == "__main__":
+    req_params = required_env_var("BINANCE_CREATE_ORDER_PARAMS")
+    res = expand_var(req_params, "SYMBOL", "BTCUSDT")
+    res = expand_defaults(res)
+    print(res)

tools/binance/get_binance_md.py

@@ -1,295 +0,0 @@
-import argparse
-import hmac
-import hashlib
-import os
-import dotenv
-from base64 import b64encode
-import logging
-import sys
-import time
-import websocket
-import json
-import requests
-from threading import Thread
-from typing import Callable, Optional, TextIO
-
-
-def hmac_hashing(api_secret: str, payload: str):
-    m = hmac.new(api_secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256)
-    return m.hexdigest()
-
-
-def rest_request(base_url: str, api_key: str, api_secret: str, parameters: dict = {}):
-    parameters["timestamp"] = int(time.time() * 1000)
-    sorted_keys = list(parameters)
-    sorted_keys.sort()
-    query_params="&".join([f"{k}={parameters[k]}" for k in sorted_keys])
-    signature = hmac_hashing(api_secret=api_secret, payload=query_params)
-    parameters["signature"] = signature
-    query_params += "&signature=" + signature
-    url = base_url + "/api/v3/userDataStream" + "?" + query_params
-    res = requests.post(url, headers={"X-MBX-APIKEY": api_key})
-    return res.text
-
-
-def get_listen_key(base_url: str, api_key: str):
-    url = base_url + "/api/v3/userDataStream"
-    res = requests.post(url, headers={"X-MBX-APIKEY": api_key})
-    return res.text
-
-
-class CircularBufferWriter:
-    def __init__(self, buffer_line_count: int, sink: TextIO = sys.stdout):
-        self.buffer_lines = []
-        self.buffer_line_count = buffer_line_count
-        self.sink = sink
-    
-    def print(self, msg):
-        if self.buffer_line_count <= 0:
-            self.sink.write(msg + '\n')
-            return
-        for _ in range(0, len(self.buffer_lines)):
-            self.sink.write("\x1b[1A\x1b[2K") # move up cursor and delete whole line
-        if len(self.buffer_lines) == self.buffer_line_count:
-            self.buffer_lines.pop()
-        self.buffer_lines.insert(0, msg)
-        lines = ''
-        num_lines = len(self.buffer_lines)
-        for i in range(0, num_lines):
-            lines += self.buffer_lines[num_lines - i - 1] + '\n'
-        self.sink.write(lines)
-
-
-def book_ticker_stream():
-    return "bookTicker"
-
-def depth_stream(symbol: str, num_levels: Optional[int] = 5, interval_ms: Optional[int] = 500):
-    rv = f"{symbol}@depth"
-    if num_levels is not None:
-        rv += str(num_levels)
-    if interval_ms is not None:
-        rv += f"{interval_ms}ms"
-    return rv
-
-class OrderBookSubscription:
-    def __init__(
-        self,
-        msg_handler: Callable,
-        streams: list[str],
-        url: str = "wss://stream.binancefuture.com:443/ws",
-        logger: logging.Logger = logging.getLogger(),
-    ):
-        self.msg_handler = msg_handler
-        self.streams = streams
-        self.url = url
-        self.logger = logger
-
-    def on_open(self, ws: websocket.WebSocketApp):
-        def run(*args):
-            try:
-                req = {
-                    "method": "SUBSCRIBE",
-                    "params": self.streams,
-                    "id": 1,
-                }
-                req_str = json.dumps(req)
-                self.logger.info("Send: %s", req_str)
-                ws.send_text(req_str)
-            except Exception as ex:
-                self.logger.error("Error: %s", ex)
-                self.logger.info("Disconnecting websocket")
-                ws.close()
-
-        self.logger.info("Websocket connected")
-        th = Thread(target=run)
-        th.start()
-        th.join()
-
-    def on_message(self, ws, msg):
-        self.logger.debug("Recv: %s", msg)
-        self.msg_handler(msg)
-
-    def on_error(self, ws, err):
-        self.logger.error("Error: %s", err)
-
-    def on_close(self, ws, code, msg):
-        self.logger.info("Closed (%d): %s", code, msg)
-
-    def run(self):
-        ws_app = websocket.WebSocketApp(
-            url=self.url,
-            on_open=self.on_open,
-            on_message=self.on_message,
-            on_error=self.on_error,
-            on_close=self.on_close,
-        )
-        ws_app.run_forever(ping_interval=600, ping_timeout=180)
-
-class UserDataSubscription:
-    def __init__(
-        self,
-        msg_handler: Callable,
-        url: str,
-        logger: logging.Logger = logging.getLogger(),
-    ):
-        self.msg_handler = msg_handler
-        self.url = url
-        self.logger = logger
-
-    def on_open(self, ws: websocket.WebSocketApp):
-        self.logger.info("Websocket connected")
-
-    def on_message(self, ws, msg):
-        self.logger.debug("Recv: %s", msg)
-        self.msg_handler(msg)
-
-    def on_error(self, ws, err):
-        self.logger.error("Error: %s", err)
-
-    def on_close(self, ws, code, msg):
-        self.logger.info("Closed (%d): %s", code, msg)
-
-    def run(self):
-        ws_app = websocket.WebSocketApp(
-            url=self.url,
-            on_open=self.on_open,
-            on_message=self.on_message,
-            on_error=self.on_error,
-            on_close=self.on_close,
-        )
-        ws_app.run_forever(ping_interval=600, ping_timeout=180)
-
-
-def _main():
-    # Urls
-    FUTURES_WS_URL_PROD = "wss://fstream.binance.com:443/ws"
-    FUTURES_WS_URL_DEV = "wss://stream.binancefuture.com:443/ws"
-    FUTURES_REST_URL_PROD = "https://fapi.binance.com"
-    FUTURES_REST_URL_DEV = "https://testnet.binancefuture.com"
-    SPOTS_WS_URL_DEV = "wss://stream.testnet.binance.vision:9443/ws"
-    SPOTS_WS_URL_PROD = "wss://stream.binance.com:9443/ws"
-    SPOTS_REST_URL_DEV = "https://testnet.binance.vision"
-    SPOTS_REST_URL_PROD = "https://api.binance.com"
-
-    # Cli Args
-    arg_parser = argparse.ArgumentParser(
-        description="Request market data at Binance"
-    )
-    arg_parser.add_argument(
-        "--streams",
-        help="Streams to subscribe (e.g, btcusdt@depth5@500ms)",
-    )
-    arg_parser.add_argument(
-        "--user-data",
-        action="store_true",
-        help="User data",
-    )
-    arg_parser.add_argument("--exch-info", action="store_true", help="Get exchangeInfo (REST)")
-    arg_parser.add_argument("--list-symbols", action="store_true", help="Get symbols (REST)")
-    arg_parser.add_argument("--list-streams", action="store_true", help="List all streams")
-    arg_parser.add_argument("--display-lines", help="Cap the number of lines to display from incoming messages (applies circular buffer to console output).")
-    arg_parser.add_argument("--url", help="The url to send requests to")
-    args = arg_parser.parse_args()
-
-    if args.list_streams:
-        stream_descs = [
-            "{symbol}@aggTrade",
-            "{symbol}@markPrice",
-            "{symbol}@markPrice@{interval}                    (intervals: 1s)",
-            "!markPrice@arr",
-            "!markPrice@arr@1s",
-            "{symbol}@kline_{interval}                        (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)",
-            "{pair}_{contractType}@continuousKline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)"
-            "{symbol}@miniTicker",
-            "!ticker@arr",
-            "{symbol}@ticker",
-            "!miniTicker@arr",
-            "{symbol}@bookTicker",
-            "!bookTicker",
-            "{symbol}@forceOrder",
-            "!forceOrder@arr",
-            "{symbol}@depth{levels}                           (levels: 5, 10, 20)",
-            "{symbol}@depth{levels}@{interval}                (levels: 5, 10, 20)",
-            "                                                 (intervals: 100ms, 500ms)",
-            "{symbol}@depth",
-            "{symbol}@depth@{interval}                        (intervals: 100ms, 500ms)",
-            "{symbol}@compositeIndex",
-            "!contractInfo",
-            "!assetIndex@arr",
-            "{assetSymbol}@assetIndex",
-        ]
-        print("\n".join(stream_descs))
-    elif args.exch_info:
-        # Get exch info
-        url = args.url or (FUTURES_REST_URL_DEV + "/fapi/v1/exchangeInfo")
-        res = requests.get(url, headers={"Content-Type": "application/json"})
-        if res.status_code < 200 or res.status_code > 300:
-            sys.stderr.write(f"Request failed. code: {res.status_code}, reason: {res.reason}\n")
-            return 1
-        print(res.text)
-    elif args.list_symbols:
-        # Get symbols
-        url = args.url or (FUTURES_REST_URL_DEV + "/fapi/v1/exchangeInfo")
-        res = requests.get(url, headers={"Content-Type": "application/json"})
-        if res.status_code < 200 or res.status_code > 300:
-            sys.stderr.write(f"Request failed. code: {res.status_code}, reason: {res.reason}\n")
-            return 1
-        syms = {}
-        res_j = json.loads(res.text)
-        for sym in res_j["symbols"]:
-            syms[sym["symbol"]] = sym
-        sym_list = list(syms)
-        sym_list.sort()
-        for sym in sym_list:
-            print(sym)
-    elif args.user_data:
-        logger = logging.getLogger()
-        env_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "binance.env")
-        dotenv.load_dotenv(env_path)
-        api_key = os.environ.get("BINANCE_DEV_API_KEY")
-        secret_key = os.environ.get("BINANCE_DEV_SECRET_KEY")
-        listen_key_res = get_listen_key(SPOTS_REST_URL_DEV, api_key)
-        listen_key = json.loads(listen_key_res).get("listenKey")
-        if not listen_key:
-            logger.error("Failed to get listen key")
-            return 1
-        url = args.url or SPOTS_WS_URL_DEV
-        url += "/ws/" + listen_key
-        writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
-
-        def on_message(ws, msg):
-            logger.debug("on_message: %s", msg)
-            writer.print(msg)
-
-        def on_error(ws, err):
-            logger.error("on_error: %s", err)
-
-        def on_close(ws, code, msg):
-            logger.info("on_close (%d): %s", code, msg)
-
-        ws_app = websocket.WebSocketApp(
-            url=url,
-            on_message=on_message,
-            on_error=on_error,
-            on_close=on_close,
-        )
-        ws_app.run_forever(ping_interval=600, ping_timeout=180)
-    elif args.streams:
-        # Subscribe streams
-        url = args.url or FUTURES_WS_URL_DEV
-        writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
-        sub = OrderBookSubscription(
-            msg_handler=writer.print,
-            url=url,
-            streams=args.streams if isinstance(args.streams, list) else [args.streams],
-        )
-        sub.run()
-    else:
-        sys.stderr.write("No data requested (run with --streams <stream1> <stream2> ... or --exch-info or --list-symbols)")
-        return 1
-    return 0
-
-
-if __name__ == "__main__":
-    code = _main()
-    exit(code)

tools/binance/get_binance_md.sh

@@ -1,141 +0,0 @@
-#!/usr/bin/env bash
-
-help_text="\
-Get Binance market data.
-
-Usage: get_binance_md.sh [OPTIONS]
-
-Options:
-  --usds-future     Indicates the symbol is a USDS Future
-  --coin-future     Indicates the symbol is a Coin Future
-  --spot            Indicates the symbol is a Spot
-  --us              Indicates the symbol is a Binance US Spot
-  --prod            Use production url
-  --dev             Use development url
-  --dry             Print the command to console without actually running it
-  --data            DEPTH | RECENT_TRADES | HISTORICAL_TRADES | MARK_PRICE | PRICE_TICKER
-  --<key>=<value>   Query parameter (e.g., --symbol=btcusdt, --limit=5)
-
-"
-
-function print_help_and_exit() {
-  echo "$help_text"
-  exit "$1"
-}
-
-declare this_dir="$(readlink -f $(dirname ${BASH_SOURCE[0]}))"
-source "$this_dir/binance.env"
-
-env_name="binance"
-venue="BINANCE"
-declare -u mode
-declare -u data
-declare -A query_params
-declare -i dry
-dry=0
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --usds-future)
-      venue="BINANCE_USDS_FUTURES"
-      shift 1
-      ;;
-    --coin-future)
-      venue="BINANCE_COIN_FUTURES"
-      shift 1
-      ;;
-    --spot)
-      venue="BINANCE"
-      shift 1
-      ;;
-    --us)
-      venue="BINANCE_US"
-      shift 1
-      ;;
-    --prod)
-      mode="PROD"
-      shift 1
-      ;;
-    --dev)
-      mode="DEV"
-      shift 1
-      ;;
-    --data)
-      if [[ -n "$data" ]]; then
-        echo "Unexpected argument '$1'"
-        print_help_and_exit 1
-      elif [[ -z "$2" ]]; then
-        echo "Data (--data) must be non-empty"
-        print_help_and_exit 1
-      fi
-      data="$2"
-      shift 2
-      ;;
-    --*=*)
-      key="${1%%=*}"
-      key="${key#--}"
-      value="${1#*=}"
-      if [[ -z "$key" ]]; then
-        echo "Invalid --key=value arg (key is empty). $1"
-        print_help_and_exit 1
-      fi
-      query_params["$key"]="$value"
-      shift 1
-      ;;
-    --dry)
-      dry=1
-      shift 1
-      ;;
-    -h)
-      ;&
-    --help)
-      print_help_and_exit 0
-      ;;
-    *)
-      if [[ -z "$1" ]]; then
-        echo "Invalid positional argument '$1'"
-        print_help_and_exit 1
-      elif [[ -z "$symbol" ]]; then
-        symbol="$1"
-      else
-        echo "Unexpected argument '$1'"
-        print_help_and_exit 1
-      fi
-      shift 1
-    ;;
-  esac
-done
-
-if [ -z "$mode" ]; then
-  mode="DEV"
-fi
-if [ -z "$data" ]; then
-  data="DEPTH"
-fi
-
-base_url_name="${venue}_${mode}_URL"
-base_url_name="${venue}_${mode}_URL"
-path_name="${venue}_${data}_PATH"
-params_name="${venue}_${data}_PARAMS"
-if [ ! -f "$this_dir/$env_name.env" ]; then
-  echo "Env file '$this_dir/$env_name.env' does not exist"
-  print_help_and_exit 1
-fi
-
-for key in "${!query_params[@]}"; do
-  val=${query_params[$key]}
-  declare -u keyu
-  keyu="$key"
-  eval "$keyu=$val"
-done
-
-source "$this_dir/$env_name.env"
-
-params_template=${!params_name}
-eval "params=\"$params_template\""
-url="${!base_url_name}${!path_name}?${params}"
-if [ $dry -ne 0 ]; then
-  echo curl --get $url
-else
-  curl --get $url
-fi

tools/fish/envsource.fish

@@ -0,0 +1,9 @@
+# Usage: envsource <env_file>
+
+function envsource --description "Source .env file"
+  for line in (cat $argv | grep -v '^#')
+    set item (string split -m 1 '=' $line)
+    set -gx $item[1] $item[2]
+    echo "Exported key $item[1]"
+  end
+end