fix: Gracefully handle missing column settings in pnl win

Pnl window settings are stored with the workspace on the server (like all shell
window settings), and may originate from an older version of Spark with a
different set of columns. We shouldn't crash if this happens.

.gitignore

@@ -11,6 +11,7 @@ cmake-build-debug/
 **.vscode/*
 !.vscode/launch.json
 !.vscode/tasks.json
+!.vscode/*.env
 .vs
 obj/
 *.csproj.user

.vscode/python.env

@@ -0,0 +1 @@
+PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins

CMakeLists.txt

@@ -117,7 +117,7 @@ else()
   if (BUILD_SPARKD)
     add_subdirectory(src/lib_python_interop)
     add_subdirectory(src/lib_onixs_fix)
-    add_subdirectory(src/lib_binance_execution)
+    add_subdirectory(src/lib_binance)
     add_subdirectory(src/lib_okx_execution)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)

compose.yaml

@@ -22,6 +22,10 @@ services:
           . /home/bts/sparkdata/conf/.env.local
           set +a
         fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
         ./sparkd run"
     env_file:
       - .env
@@ -61,6 +65,10 @@ services:
           . /home/bts/sparkdata/conf/.env.local
           set +a
         fi
+        if [ -d /home/bts/sparkdata/conf/plugins ]; then
+          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
+          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
+        fi
         ./sparkd run"
     env_file:
       - .env

spark_client.sln

@@ -18,6 +18,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\clie
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -90,6 +98,38 @@ Global
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
 		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
+		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -0,0 +1,11 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   [DataStreamUserRespresentation("AddColumnsStream")]
+   public class AddColumnsStream: ViewStream {
+      public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
+         ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -1,7 +1,7 @@
 using System;
 
 namespace BCG {
-   [DataStreamUserRespresentation("count")]
+   [DataStreamUserRespresentation("CountStream")]
    public class CountStream : ViewStream {
       private DataPacket myLastFiredDataPacket;
       private IBlueTableSchema mySchema = new BlueTableSchema("count",

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -3,36 +3,34 @@ namespace bts.BlueOrb {
    using System;
    using System.Text.RegularExpressions;
 
-
-   public class RegExpFilter : IStreamFilter {
-      private Regex[] myRegExps;
-      private string[] myCols;
-
-      public RegExpFilter(string col, string regExp)
-         : this(new string[] { col }, new string[] { regExp }) { }
-
-      public RegExpFilter(string[] cols, string[] regExps) {
-         myCols = cols;
-         myRegExps = new Regex[regExps.Length];
-         for (int i = 0; i < regExps.Length; i++)
-            myRegExps[i] = new Regex(regExps[i]);
-      }
-
-      #region IStreamFilter Members
-      public bool filter(IBlueRow dr) {
-         bool rv = true;
-         for (int i = 0; i < myCols.Length; i++) {
-            rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
-            if (!rv)
-               return rv;
-         }
-         return rv;
-      }
-      #endregion
-   }
-
-   //[DataStreamUserReprsentation("FilterStream")]
+   [DataStreamUserRespresentation("FilterStream")]
    public class FilterStream : ViewStream {
+      public class RegExpFilter : IStreamFilter {
+         private Regex[] myRegExps;
+         private string[] myCols;
+
+         public RegExpFilter(string col, string regExp)
+            : this(new string[] { col }, new string[] { regExp }) { }
+
+         public RegExpFilter(string[] cols, string[] regExps) {
+            myCols = cols;
+            myRegExps = new Regex[regExps.Length];
+            for (int i = 0; i < regExps.Length; i++)
+               myRegExps[i] = new Regex(regExps[i]);
+         }
+
+         #region IStreamFilter Members
+         public bool filter(IBlueRow dr) {
+            bool rv = true;
+            for (int i = 0; i < myCols.Length; i++) {
+               rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
+               if (!rv)
+                  return rv;
+            }
+            return rv;
+         }
+         #endregion
+      }
       private BlueTable myLastFiredRows;
       private IBlueRowCollection myLastReceivedRows;
       private IStreamFilter myFilter;

src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs

@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using BCG;
-
-namespace bts.BlueOrb {   
-   public interface INonIncJoinInlineMapper
-   {      
-      IBlueTableSchema get_output_schema(IBlueTableSchema schema);
-
-      IBlueRowCollection map_rows(IBlueRowCollection rows);
-      
-      event MapperChangeDelegate onMapperChange;
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs

@@ -1,152 +0,0 @@
-using bts;
-using bts.collections;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Cryptography;
-using BCG;
-
-namespace bts.BlueOrb {   
-   internal class NonIncJoinBookkeeper : IDisposable {
-      public Action on_new_data;
-      private readonly NonIncJoinSpec _join_spec;
-      private readonly int _join_position;
-      private readonly int _start_index_in_output;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly bool _is_left_most;
-      private readonly bool _is_tag_along;
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-
-      private IBlueTableSchema _schema = null;
-      private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
-      private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
-      private IBlueRow _solitary_row = null;
-
-      private int[] _join_col_indices = null,
-                     _data_col_indices = null;
-
-      public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
-         _join_spec = the_join_spec;
-         _join_position = join_position;
-         _start_index_in_output = start_index_in_output;
-         _row_hasher = row_hasher;
-         _is_left_most = _join_position == 0;
-         _is_tag_along = _join_spec.join_col_names.Length == 0;
-      }
-
-      private bool _filled_missing_data = false;
-      private void _fill_missing_data(DataPacket dp) {
-         if (_filled_missing_data)
-            return;
-         Func<string[], bool> null_or_empty = (str_arr) => {
-            if (str_arr == null || str_arr.Length == 0)
-               return true;
-            foreach (string s in str_arr) {
-               if (!String.IsNullOrEmpty(s)) {
-                  return false;
-               }
-            }
-            return true;
-         };
-         Func<string[], string[]> remove_bad_cols = (col_names) => {
-            HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
-            List<string> cols_list = new List<string>(col_names);
-            cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
-            return cols_list.ToArray();
-         };
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
-         if (null_or_empty(_join_spec.data_col_names)) {
-            _join_spec.data_col_names = dp.schema.columnNames;
-         }
-         if (null_or_empty(_join_spec.projected_col_names)) {
-            _join_spec.projected_col_names = _join_spec.data_col_names;
-         }
-      }
-
-      public void init() {
-         _join_spec.stream.onChange += _on_stream_cb;
-      }
-
-      public void _on_stream_cb(object dp_obj) {
-         //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
-         var timer = Stopwatch.StartNew();
-         DataPacket dp = (DataPacket)dp_obj;
-         _fill_missing_data(dp);
-         if (_schema == null) {
-            _schema = dp.schema;
-            _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
-            _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
-         }
-
-         if (_is_left_most) {
-            _row_list.Clear();
-            foreach (IBlueRow r in dp.newRows)
-               _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
-         } else {
-            if (_is_tag_along)
-               _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
-            else {
-               _row_map.Clear();
-               foreach (IBlueRow r in dp.newRows)
-                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
-            }
-         }
-         timer.Stop();
-         _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
-            _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
-         }
-         if (on_new_data != null)
-            on_new_data();
-      }
-
-      public bool is_ready { get { return _schema != null; } }
-
-      public void constribute_cols_to_schema(List<ColumnInfo> cis) {
-         for (int i = 0; i < _join_spec.data_col_names.Length; i++)
-            cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
-      }
-
-      public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
-
-      public void copy_master_row_to_output_row(int row_index, object[] output_row) {
-         IBlueRow this_row = _row_list[row_index].second;
-         BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-      }
-
-      public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
-         if (_is_tag_along) {
-            if (_solitary_row != null)
-               BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         } else {
-            IBlueRow this_row = null;
-            if (_row_map.TryGetValue(row_hash, out this_row))
-               BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
-            else
-               copy_dbnull_to_output_row(output_row);
-         }
-      }
-
-      private void copy_dbnull_to_output_row(object[] output_row) {
-         for (int i = 0; i < _data_col_indices.Length; i++)
-            output_row[i + _start_index_in_output] = DBNull.Value;
-      }
-
-      #region IDisposable
-
-      public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs

@@ -1,67 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using BCG;
-
-using mbts;
-
-namespace bts.BlueOrb {   
-   // this a collection of spec, theres no joining logic in here
-   public class NonIncJoinSpec {
-      public ViewStream stream;
-      // join col names are the names on which the streams are joined (for equality),
-      // data col names are the names in the original stream
-      // projected col names are their new names, ie if its x:l_x, then x is in the 
-      // data_col_names and l_x in projected cols
-      public string[] join_col_names, data_col_names, projected_col_names;
-
-      public NonIncJoinSpec() { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
-         : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
-
-      public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
-         stream = the_stream;
-         join_col_names = the_join_col_names;
-         data_col_names = the_data_col_names;
-         projected_col_names = the_projected_col_names;
-      }
-
-      public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
-         List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
-         foreach (Triplet<ViewStream, string, string> r in join_specs) {
-            string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
-            string[] visible_cols = r.third.Split(' ');
-            List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
-            foreach (string s in visible_cols) {
-               string[] toks = s.Split(':');
-               if (toks.Length == 1) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[0]);
-               } else if (toks.Length == 2) {
-                  data_col_names.Add(toks[0]);
-                  mapped_col_names.Add(toks[1]);
-               }
-            }
-            rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
-         }
-         return rv.ToArray();
-      }
-
-      public override string ToString() {
-         Func<string[], string> arr_txt = (a) => {
-            string rv = "";
-            foreach (string s in a) {
-               rv += "{s} ";
-            }
-            rv = rv.Trim();
-            return rv;
-         };
-         string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
-         return rv;
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs

@@ -1,165 +0,0 @@
-using bts;
-
-using mbts;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using BCG;
-
-namespace bts.BlueOrb {   
-   [DataStreamUserRespresentation("JoinStream")]
-   public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
-      private readonly NonIncJoinSpec[] _join_specs;
-      private readonly INonIncJoinInlineMapper _output_mapper;
-      private readonly BlueRowHasher _row_hasher;
-      private readonly TimeSpan _throttle_period;
-      private readonly ViewStreamDatabase _vsd;
-
-      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
-      private IBlueTableSchema _output_schema = null;
-      private DateTime _last_fire_time = DateTime.MinValue;
-      private bool _is_dirty = false;
-
-      public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
-                              ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[]
-               {
-                  new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
-                  new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
-               }) { }
-
-      public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
-                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { 
-      }
-
-      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
-         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
-
-      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
-         _join_specs = join_specs;
-         _output_mapper = output_mapper;
-         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
-         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
-         _vsd = vsd;
-      }
-
-      protected override void myOnFirstSubscriber() {
-         int data_col_no = 1;
-         for (int jp = 0; jp < _join_specs.Length; jp++) {
-            NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
-            //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
-            data_col_no += _join_specs[jp].data_col_names.Length;
-            s.on_new_data += _stream_callback;
-            _book_keeping_data.Add(s);
-         }
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange += _on_mapper_change;
-         _vsd.addWorker(this);
-         foreach (var bkd in _book_keeping_data)
-            bkd.init();
-      }
-
-      protected override void myOnLastSubscriber() {
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            d.Dispose();
-         if (_output_mapper != null)
-            _output_mapper.onMapperChange -= _on_mapper_change;
-         _vsd.removeWorker(this);
-      }
-
-      private void _stream_callback() {
-         //Log.Gen.info("NonIncJoinStream._stream_callback received callback");
-         _is_dirty = true;
-         doWork();
-      }
-
-      private void _on_mapper_change() {
-         _is_dirty = true;
-         doWork();
-      }
-
-      private bool _is_ready() {
-         bool rv = true;
-         foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-            rv &= d.is_ready;
-         return rv;
-      }
-
-      private void _construct_schema() {
-         if (_output_schema == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
-            foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema(cis);
-            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
-         }
-      }
-
-      private int _n_total_fires = 0, _n_expensive_fires = 0;
-      private void _form_and_fire() {
-         //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
-         if (_is_ready()) {
-            _construct_schema();
-            var timer = Stopwatch.StartNew();
-            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
-            List<object[]> output_rows = new List<object[]>();
-
-            for (int i = 0; i < left_row_list.Count; i++) {
-               object[] or = new object[_output_schema.columnInfo.Length];
-               or[0] = i;
-               output_rows.Add(or);
-            }
-
-            for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
-               NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
-               for (int i = 0; i < left_row_list.Count; i++) {
-                  if (ji == 0)
-                     bk_data.copy_master_row_to_output_row(i, output_rows[i]);
-                  else
-                     bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
-               }
-            }
-            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
-            _n_total_fires++;
-            //if (timer.ElapsedMilliseconds >= 250) {
-            _n_expensive_fires++;
-            //string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-            //   + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
-            //Log.Gen.info(log_msg);
-            //}
-            var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
-            myFireEvent(dp_out);
-            //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
-         }
-      }
-
-      private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
-         if (_output_mapper != null) {
-            IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
-            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
-            }
-            return output_rows;
-         } else {
-            return unmapped_rows;
-         }
-      }
-
-      public void doWork() {
-         //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
-         if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
-            _is_dirty = false;
-            _last_fire_time = DateTime.Now;
-            _form_and_fire();
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -0,0 +1,187 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using RowHash = System.Int32;
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   using System.Collections;
+   using System.Data.Entity.Core.Mapping;
+
+   [DataStreamUserRespresentation("JoinStream")]
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
+         // initialized in cons
+         private readonly List<string> _join_col_names;
+         private readonly bool _is_left;
+         private readonly BlueRowHasher _hasher;
+         // initialized on first udpate         
+         private IBlueTableSchema _schema;
+         private int[] _join_cols;
+         private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
+         
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
+            _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
+            _is_left = is_left;
+            _hasher = row_hasher;
+         }
+
+         public IBlueTableSchema schema => _schema;
+
+         public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
+
+         public List<IBlueRow> get_matching_rows (int hash) {
+            List<IBlueRow> rv = null;
+            _hash_row_map.TryGetValue(hash, out rv);
+            return rv;
+         }
+
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) { 
+               return 0;
+            }
+            return _hasher.get_hash(r, _join_cols);
+         }
+
+         public void update_data(DataPacket dp) {
+            if (!ready) {               
+               _schema = dp.newRows.schema;
+               _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
+            }
+            _hash_row_map.Clear();
+            foreach (IBlueRow r in dp.newRows) {
+               int hash = _get_hash(r);
+               List<IBlueRow> hash_rows = null;
+               if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
+                  hash_rows = new List<IBlueRow>();
+                  _hash_row_map.Add(hash, hash_rows);
+               }
+               hash_rows.Add(r);
+            }
+         }
+
+         public bool ready {
+            get {
+               return _schema != null;
+            }
+         }
+
+      }
+
+      private ViewStream _ls, _rs;
+      private JoinBookkeepingData _ldata, _rdata;
+      private ViewStreamDatabase _vsd;
+      private IBlueTableSchema _schema = null;
+
+      public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
+
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
+         _ls = ls;
+         _rs = rs;
+         List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
+         Log.assert(lj_col_list.Count ==  rj_col_list.Count);
+         int n_cols = lj_col_list.Count;
+         BlueRowHasher hasher = new BlueRowHasher(n_cols);
+         _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
+         _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _ls.onChange += _ls_onchange;
+         _rs.onChange += _rs_onchange;
+      }
+
+      private void _ls_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _ldata.update_data(dp);
+         _form_and_fire();
+      }
+
+      private void _rs_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket ;
+         _rdata.update_data(dp);
+         _form_and_fire();
+      }
+
+      protected override void myOnLastSubscriber() {
+         _ls.onChange -= _ls_onchange;
+         _rs.onChange -= _rs_onchange;
+      }
+
+      private void _build_schema() {
+         if (_schema != null)
+            return;
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
+         foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
+            string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
+            string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
+            cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
+         }
+         _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
+      }
+
+      private void _form_and_fire() {
+         if (!_ldata.ready||!_rdata.ready)
+            return;
+         _build_schema();
+         List<object[]> res_rows = new List<object[]>();
+         
+         Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
+            int offset = 1;
+            JoinBookkeepingData sdata = _ldata;
+            if (is_left) {
+               offset = 1;
+               sdata = _ldata;
+            } else {
+               sdata = _rdata;
+               offset = 1 + _ldata.schema.columnInfo.Length;
+            }
+            for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
+               rval[offset + i] = r[i];
+            }
+         };
+         Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
+            object[] rv = new object[_schema.columnNames.Length];            
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
+               rv[i] = DBNull.Value;
+            }
+            rv[0] = id;
+            copy_row(rv, l_row, true);
+            return rv;
+         };
+         // this is where the join happens
+         foreach (var kv in _ldata.rows) {
+            RowHash hash = kv.Key;
+            List<IBlueRow> l_rows = kv.Value;
+            List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
+
+            foreach (IBlueRow lr in l_rows) {                              
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
+                     object[] rowvals = build_row_array(res_rows.Count, lr);
+                     copy_row(rowvals, r, false);
+                     res_rows.Add(rowvals);
+                  }
+               } else {
+                  object[] rowvals = build_row_array(res_rows.Count, lr);
+                  res_rows.Add(rowvals);
+               }
+            }
+         }
+         BlueTable rv = new BlueTable(_schema);
+         foreach (object[] rval in res_rows) {
+            rv.addRow(new BlueRow(_schema, rval));
+         }
+         var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs

@@ -49,14 +49,14 @@ namespace bts.BlueOrb {
          }
       }
    }
-   public class MSecondStream : TimeUnitStream {
+   internal class MSecondStream : TimeUnitStream {
       public MSecondStream(int intervalInms, int size)
          : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
 
       protected override DateTime myRoundTime(DateTime dt) { return dt; }
    }
 
-   public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
+   internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
       private IBlueTableSchema mySchema = new BlueTableSchema("ms",
                                                                new ColumnInfo("id", 0, typeof(int)),
                                                                new ColumnInfo("time", 1, typeof(string)));

src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs

@@ -20,13 +20,14 @@ namespace bts.BlueOrb {
       Sum,
       EdgeDiff,
       EdgeDiffPercentage,
-      //IsEdgeDiff,
    }
    public class MultiRowOpHelper {
+      // input + op
       public readonly List<Tuple<string, MultiRowOp>> operations;
 
       public static MultiRowOpHelper parse_ops(string text) {
          // text is inp_col: op, inp_col: op, inp_col:op
+         // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
          List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
          string[] calcs = text.Split(',');
          foreach (string c in calcs) {
@@ -71,6 +72,35 @@ namespace bts.BlueOrb {
          return or;
       }
 
+      // aggregates multiple rows into one, used by MultiRowRollingMathStream
+      public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
+         if (rows.Count() == 0) {
+            return null;
+         }
+         IBlueRow ir = rows.ElementAt(start_idx);
+         IBlueTableSchema in_schema = ir.schema;
+         IBlueTableSchema out_schema = in_schema;
+         // out_col vs inp_cal + type
+         IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
+            new Dictionary<string, Tuple<string, MultiRowOp>>();
+         foreach (var t in parameters.operations) {
+            string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
+            out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
+            out_col_map[out_col_name] = t;
+         }
+         MutableBlueRow or = new MutableBlueRow(out_schema, ir);
+         foreach (var kv in out_col_map) {
+            string out_col_name = kv.Key;
+            string in_col_name = kv.Value.Item1;
+            MultiRowOp op = kv.Value.Item2;
+            IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
+            double comp_res = compute_op(col_vals, op);
+            or.setValue(out_col_name, comp_res);
+            or.setValue("id", ir["id"]);
+         }
+         return or;
+      }
+
       // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
       // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
 
@@ -107,9 +137,13 @@ namespace bts.BlueOrb {
             case MultiRowOp.Min:
                return col_vals.Min<double>();
             case MultiRowOp.Mean:
-               return col_vals.Sum() / col_vals.Count();
+               return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
             case MultiRowOp.Sum:
                return col_vals.Sum();
+            case MultiRowOp.Median:
+               return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
+            //case MultiRowOp.Mode:
+            //   return MathNet.Numerics.Statistics.Statistics.(col_vals);
             case MultiRowOp.First:
                return col_vals.First();
             case MultiRowOp.Last:

src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs

@@ -0,0 +1,40 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Net;
+
+   [DataStreamUserRespresentation("MultiRowRollingMathStream")]
+   public class MultiRowRollingMathStream : ViewStream {
+      private ViewStream _sorted_parent_str;
+      private MultiRowOpHelper _op;
+      private int _rolling_window_sz;
+      public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
+         _sorted_parent_str = new SortStream(parent_str, sort_exp);
+         _op = MultiRowOpHelper.parse_ops(op_txt);
+         _rolling_window_sz = rolling_window_sz;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _sorted_parent_str.onChange += _parent_str_onchange;
+      }      
+
+      protected override void myOnLastSubscriber() {
+         _sorted_parent_str.onChange -= _parent_str_onchange;
+      }
+
+      private void _parent_str_onchange(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         var rows = dp.newRows.to_list();
+         BlueTable out_table = null;
+         for (int i = 0; i < rows.Count; i++) {
+            var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
+            if (out_table == null)
+               out_table = new BlueTable(out_r.schema);
+            out_table.addRow(out_r);
+         }            
+         var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
+         myFireEvent(dp_out);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs

@@ -1,8 +1,5 @@
 namespace bts.BlueOrb {
    using BCG;
-
-   using ProtoBuf;
-
    using System;
    using System.Collections;
    using System.Collections.Generic;
@@ -26,7 +23,7 @@ namespace bts.BlueOrb {
    #endregion
 
 
-#region Row with mapipings build in
+#region Row with mappings build in
    internal class NewProjectRow : IBlueRow {
       #region IBlueRow members
       private ProjectData _project_data;
@@ -135,38 +132,30 @@ namespace bts.BlueOrb {
          return new NewProjectRow(_project_data, row);
       }
 
+      // this is where most of the logic is, in terms of translating cols
       private static ProjectData _create_project_data(IBlueTableSchema input_schema,
                                                       string project_expression) {
          project_expression = project_expression.Trim();
-         string[] project_toks = project_expression.Split(' ');
+         string[] project_toks = project_expression.Split(' ', ',');
          HashSet<string> removed_cols = new HashSet<string>();
-         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
-         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();      
          // by default every col will show up, so if we have the projection as (a:a1)
          // it will convert a to a1, 
+         // modifications (6/30/2025):
+         // only the columns represented here, either mapped
+         // or not, will show
          // examples: if a schema has cols a b c
-         // project(parent_str, "a:aa ") -> aa b c
-         // project(parent_str, "a:aa !c") -> aa b
-         // project(parent_str, "a:aa !") -> aa
-         // project(parent_str, "a:aa b")) -> aa b
-         bool hide_all = false;
-         foreach (string p_tok in project_toks) {
-            if (p_tok.StartsWith("!")) {
-               if (p_tok.Length == 1) {
-                  // if it is a "blanket "!"=> hide all cols except the one that
-                  // are stated explicitly
-                  hide_all = true;
-               } else {
-                  string removed_col = p_tok.Substring(1);
-                  removed_cols.Add(removed_col);
-               }
-            } else if (p_tok.Contains(":")) {
+         // project(parent_str, "a:aa b") -> aa b
+         foreach (string o_p_tok in project_toks) {
+            string p_tok = o_p_tok.Trim();
+            if (p_tok.Contains(":")) {
+               // explicit mapping
                string[] map_toks = p_tok.Split(":");
                string orig_col = map_toks[0];
                string mapped_col = map_toks[1];
-               mapped_to_orig_map.Add(mapped_col, orig_col);
-               orig_to_mapped.Add(orig_col, mapped_col);
+               orig_to_mapped.Add(orig_col, mapped_col);  
             } else {
+               // this is a col_name explicitely stated
                if (input_schema.columnNames.Contains(p_tok)) {
                   string orig_col = p_tok;
                   string mapped_col = orig_col;
@@ -181,27 +170,19 @@ namespace bts.BlueOrb {
             if (ci.colName == "id") {
                ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
                new_cols.Add(nc);
-               continue;
-            }
-            if (removed_cols.Contains(ci.colName)) {
-               continue;
+               continue;           
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
                string mapped_name = orig_to_mapped[ci.colName];
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
-            } else {
-               if (hide_all)
-                  continue;
-               //if (ci.colName == "id" || removed_cols.Count > 0) {
-               // unless columns are  explicitely hidden, all columns show up
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               //}
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
          return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
       }
 
+   
+
    }
 }

src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs

@@ -0,0 +1,127 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using System.IO;
+   using System.Diagnostics;
+   using System;
+
+   [DataStreamUserRespresentation("PythonDataframeStream")]
+   public class PythonDataframeStream : ViewStream {
+      private ViewStream _parent_str;
+      private readonly string _in_csv_file_name;
+      private readonly string _out_csv_filename;
+      private readonly string _transform_function;
+      public PythonDataframeStream(ViewStream parent_str, string transform_function) {
+         var guid = System.Guid.NewGuid();
+         _in_csv_file_name = $"pdfs_in_{guid}.csv";
+         _out_csv_filename = $"pdfs_out_{guid}.csv";
+         _parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
+         _transform_function = transform_function;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _parent_str.onChange += _parent_str_onchange;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _parent_str.onChange -= _parent_str_onchange;
+      }
+
+      private void _parent_str_onchange(object dp_obj) {
+         Log.assert(dp_obj != null);
+         // the file _in_csv_file_name is now populated.
+         // call py_func_wrapper_main and pass this file name as well
+         // as the output file name
+         // py_func_wrapper_main reads _in_csv_file_name into the df and calls 
+         // transform function, which returns the tranformed df. py_func_wrapper_main then
+         // writes it into the _out_csv_filename file.
+         // now this function should read _out_csv_filename into a dp
+         // and fire it
+         _compute_and_write();
+         ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
+         IBlueRowCollection nrows = null;
+         rs.onChange += (dp_obj) => {
+            nrows = (dp_obj as DataPacket).newRows;
+         };
+         var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      private void _compute_and_write() {
+         File.WriteAllText("user_code.py", _transform_function);
+         // wrapper function
+         string py_func_wrapper_main = @"
+import sys
+from user_code import transform
+import pandas as pd
+
+if __name__ == '__main__':
+   input_file_name = sys.argv[1]
+   output_file_name = sys.argv[2]
+# 1. load input_file_name into df
+# 2. do some test calc
+# 3. write the output of 2 into output_file_name
+   df = pd.read_csv(input_file_name)   
+   df_out = transform(df)
+   df_out.to_csv(output_file_name, index=False)";
+
+
+         File.WriteAllText("runner.py", py_func_wrapper_main);
+
+         ProcessStartInfo psi = new ProcessStartInfo {
+            FileName = "python",
+            Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
+            RedirectStandardOutput = false,
+            UseShellExecute = false
+         };
+         var proc = Process.Start(psi);
+         proc.WaitForExit();
+
+      }
+
+      #region Unused test code
+      private void _simple_test() {
+
+         // actual function that needs to be called ////////////////////////
+         string py_real_function = @"
+def test(x, out_path):
+   rv = x * 10
+   with open(out_path, 'w') as f:
+      f.write(str(rv))
+   return rv";
+         // the function is in a string object.
+         File.WriteAllText("user_code.py", py_real_function);
+
+
+         // runner app (top level main function) //////////////////////////////
+         string py_func_wrapper_main = @"
+import sys
+from user_code import test
+
+if __name__ == '__main__':
+   arg = int(sys.argv[1])
+   out_path = sys.argv[2]
+   rv = test(arg, out_path)
+   rv = rv + 12
+   with open(out_path, 'w') as f:to \
+      f.write(str(rv))";
+   
+         File.WriteAllText("runner.py", py_func_wrapper_main);
+
+         string outputPath = "py_out.txt";
+         ProcessStartInfo psi = new ProcessStartInfo {
+            FileName = "python",
+            Arguments = $"runner.py 4 {outputPath}",
+            RedirectStandardOutput = false,
+            UseShellExecute = false
+         };
+         var proc = Process.Start(psi);
+         proc.WaitForExit();
+
+         // Now read the output written by the function
+         string result = File.ReadAllText(outputPath);
+         Console.WriteLine(result); // Should print "6"
+
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs

@@ -0,0 +1,17 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+
+   [DataStreamUserRespresentation("SortStream")]
+   public class SortStream: ViewStream {
+      public SortStream (ViewStream parent_stream, string sort_expr) {
+         ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs

@@ -0,0 +1,16 @@
+namespace bts.BlueOrb {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Text;
+   using System.Threading.Tasks;
+   [DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
+   public class TimeBasedRollingFilterStream : ViewStream {
+      public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
+         ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -2,7 +2,7 @@ using System;
 
 using BCG;
 
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    // Keeps adding changing rows to the stream, hence preserving the history
    // and creating a time series, for example this can be used to make bbo updates
    // into a time series of prices/qty 

src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs

@@ -1,261 +1,16 @@
-using System;
-using System.Collections.Generic;
-
-using mbts;
-using BCG;
-
 namespace bts.BlueOrb {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
+   using System;
+   using System.Collections.Generic;
 
-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
+   using mbts;
+   using BCG;
 
-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
-   /// MutableStream can append multiple streams
-   /// as well as give the user the flexibility
-   /// to append (or remove) more streams at
-   /// runtime
-   /// </summary>
-   }
-namespace bts.BlueOrb { 
    [DataStreamUserRespresentation("UnionStream")]
-   public class MutableAppendStream : ViewStream {
-      private bool mySubscriptionActive = false;
-      private MutableAppendTable myTable;
-      private int myMovingIndex;
-      private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
-      private bool _allign_col_orders = true;
-
-      public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
-         : this(allign_col_order, new List<ViewStream>(streams)) { }
-
-      public MutableAppendStream(ICollection<ViewStream> streams)
-         : this(true, streams) { }
-
-      public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
-         _allign_col_orders = allign_col_orders;
-         foreach (ViewStream vs in streams) {
-            myMovingIndex++;
-            ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-            if (_allign_col_orders)
-               pj = new ProjectStream(pj, "!");
-            myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         }
-      }
-
-      public void addStream(ViewStream vs) {
-         myMovingIndex++;
-         ///the change name is done to identify the callbacks, and the project to
-         ///make all the schemas col orders the same
-         ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
-         if (_allign_col_orders)
-            pj = new ProjectStream(pj, "!");
-         myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
-         if (mySubscriptionActive)
-            pj.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      public void removeStream(ViewStream vs) {
-         int index = -1;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
-            if (p.Value.first == vs) {
-               index = p.Key;
-               if (mySubscriptionActive) {
-                  p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-                  if (myTable != null) {
-                     IBlueRowCollection delRows = myTable.removeTable(p.Key);
-                     if (delRows != null)
-                        myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
-                  }
-               }
-               break;
-            }
-            if (index > 0)
-               myStreamIndex.Remove(index);
-         }
-      }
-
-      protected override void myOnFirstSubscriber() {
-         mySubscriptionActive = true;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
-      }
-
-      protected override void myOnLastSubscriber() {
-         mySubscriptionActive = false;
-         foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
-            p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
-         if (myTable != null)
-            myTable.clear();
-      }
-
-      private void myHandleStreamChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
-         BlueTable adds = new BlueTable(myTable.schema),
-                     dels = new BlueTable(myTable.schema);
-         int index = Int32.Parse(dp.schema.tableName);
-         foreach (IBlueRow r in dp.deletedRows) {
-            IBlueRow dr = myTable.removeRow(r, index);
-            dels.addRow(dr);
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            IBlueRow nr = myTable.addRow(r, index);
-            adds.addRow(nr);
-         }
-         myFireEvent(new DataPacket(myTable, adds, dels));
+   public class UnionStream : ViewStream {
+      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
+         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
+         mySetRealSource(rs);
       }
    }
+
 }

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -2,15 +2,27 @@ namespace Bts.DataStream;
 
 using BCG;
 
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using Microsoft.Extensions.Logging;
+
 using Spark.Core.Disposables;
 
 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reactive.Linq;
 
 public static class DataStreamExtensions {
    public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
       source as ViewStream ?? new ObservableViewStream(source);
 
+   public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
+      source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
+
    public static IObservable<DataPacket> AsObservable(this ViewStream source) {
       return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
 
@@ -23,4 +35,100 @@ public static class DataStreamExtensions {
                $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
+
+   public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
+      source.Select(dp => {
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
+
+         IBlueRowCollection<T> CastRows(ICollection rows) =>
+            new BlueTableSelectorWrapper<T>(
+               rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
+      });
+
+   public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
+      source.AsTables<IBlueRow, int>(r => r, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
+      source.AsTables<T, int>(valueSelector, r => r.Id());
+
+   public static IObservable<TableUpdate<T, int>> AsTables<T>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
+      source.AsTables<T, int>(valueSelector, keySelector);
+
+   public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
+      this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
+      rowKeySelector ??= r => keySelector(valueSelector(r));
+      return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
+         var bd = res.to_builder();
+         dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
+         foreach (IBlueRow r in addedRows) {
+            bd.add_or_update(valueSelector(r));
+         }
+         foreach (IBlueRow r in deletedRows) {
+            bd.remove(rowKeySelector(r));
+         }
+         return bd.to_table();
+      });
+   }
+
+   public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.Scan(
+         (Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
+         (res, table) => {
+            var bd = res.Table.ToDataPacketBuilder();
+            foreach (var r in table.removed) {
+               bd.Remove(r.Id());
+            }
+            foreach (var r in table.added) {
+               bd.AddOrUpdate(r);
+            }
+            var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
+            return (newRows, resDp);
+         })
+         .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
+         .Select(t => t.Dp);
+
+   public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
+      source.AsDataPackets(schema).AsViewStream();
+
+   public static int ResolveIndices(this IBlueTableSchema schema, string name) {
+      var indices = ResolveIndices(schema, new[] { name });
+      return indices[0];
+   }
+
+   public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) {
+      var indices = ResolveIndices(schema, new[] { name1, name2 });
+      return (indices[0], indices[1]);
+   }
+
+   public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3 });
+      return (indices[0], indices[1], indices[2]);
+   }
+
+   public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) {
+      var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 });
+      return (indices[0], indices[1], indices[2], indices[3]);
+   }
+
+   public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) {
+      var columns = schema.columnInfo;
+      var indices = new int[columnNames.Length];
+      int nResolved = 0;
+      for (int i = 0; i < columnNames.Length; i++) {
+         int origNResolved = nResolved;
+         for (int j = 0; j < columns.Length; j++) {
+            if (columns[j].colName == columnNames[i]) {
+               indices[i] = j;
+               nResolved++;
+               break;
+            }
+         }
+         if (nResolved == origNResolved) {
+            throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema));
+         }
+      }
+      return indices;
+   }
 }

src/DataStreamSuite/DataStream/ObservableViewStream.cs

@@ -4,14 +4,21 @@ using BCG;
 
 using bts;
 
+using Microsoft.Extensions.Logging;
+
 using System;
 
 public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
    readonly IObservable<DataPacket> source;
+   readonly string? name;
+   readonly ILogger? logger;
    IDisposable? subscription;
 
-   public ObservableViewStream(IObservable<DataPacket> source) =>
+   public ObservableViewStream(IObservable<DataPacket> source, string? name = null, ILogger? logger = null) {
       this.source = source;
+      this.name = name;
+      this.logger = logger;
+   }
 
    public void Dispose() {
       GC.SuppressFinalize(this);
@@ -23,7 +30,13 @@ public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDispos
 
    protected override void myOnFirstSubscriber() {
       Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
+      subscription = source.Subscribe(myFireEvent, OnError, OnCompleted);
+
+      void OnError(Exception exception) =>
+         logger?.LogError("Error in ObservableViewStream. name: {Name}, error: {Exception}", name, exception);
+
+      void OnCompleted() =>
+         logger?.LogInformation("ObservableViewStream completed. name: {Name}", name);
    }
 
    protected override void myOnLastSubscriber() {

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,33 +1,51 @@
 using System;
+using System.Collections.Generic;
 
-namespace BCG {
+namespace BCG {   
    public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_exp(string exp) {
+      public static Tuple<string, Type, object> parse_single_tok(string exp) {
          string[] toks = exp.Split(':');
-         string col_name = toks[0];
-         string col_value = toks[1];
+         string col_name = toks[0].Trim();
+         string col_value = toks[1].Trim();
          Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
          return Tuple.Create(col_name, tup.Item1, tup.Item2);
       }
 
+      public static List<Tuple<string, Type, object>> parse_exp(string exp) {
+         string[] toks = exp.Split(",");
+         List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
+         foreach (string s in toks) {
+            var tup = parse_single_tok(s.Trim());
+            rv.Add(tup);
+         }
+         return rv;
+      }
+
       private Tuple<string, Type, object> _new_col;
+      private List<Tuple<string, Type, object>> _new_col_map;
       private IBlueTableSchema _out_schema;
 
       public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col = parse_exp(col_def);
+         _new_col_map = parse_exp(col_def);
          ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
          mySetRealSource(rs);
       }
 
       private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null)
-            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         if (_out_schema == null) {
+            _out_schema = input_schema;
+            foreach (var m in _new_col_map) {
+               _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
+            }
+         }
          return _out_schema;
       }
 
       private IBlueRow _map(IBlueRow r) {
          MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         rv.setValue(_new_col.Item1, _new_col.Item3);
+         foreach (var m in _new_col_map) {
+            rv.setValue(m.Item1, m.Item3);
+         }
          return rv;
       }
    }

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -52,14 +52,14 @@ namespace BCG {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
          myVsd = vsd;
          _time_col = rowTimeColumnName;
          _ttl = rowTimeToLive;
-         myFilterFrequency = filterFrequency;
+         _filter_period = filter_period;
          myNextFilter = DateTime.MinValue;
 
          // add worker
@@ -92,7 +92,7 @@ namespace BCG {
             // fire event
             onFilterChange();
 
-            myNextFilter = now.Add(myFilterFrequency);
+            myNextFilter = now.Add(_filter_period);
          }
       }
       #endregion
@@ -108,7 +108,7 @@ namespace BCG {
       private ViewStreamDatabase myVsd;
       private string _time_col;
       private TimeSpan _ttl;
-      private TimeSpan myFilterFrequency;
+      private TimeSpan _filter_period;
       private DateTime myNextFilter;
       #endregion
    }

src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs

@@ -8,7 +8,7 @@ using bts;
 using bts.SilverPine;
 
 namespace BCG {
-   [DataStreamUserRespresentation("AggregateStream")]
+   //[DataStreamUserRespresentation("AggregateStream")]
    public class NonIncGroupByStream : ViewStream {
       private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
          Func<string[]> _groupby_cols_func;

src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs

@@ -1,31 +1,31 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+//using System;
+//using System.Collections.Generic;
+//using System.Linq;
+//using System.Text;
 
-namespace BCG
-{
-   public class FullJoinStream : ViewStream
-   {
-      private ViewStream myMainStream;
-      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
-         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
-      {
-         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
-         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
-         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
-         myMainStream = new AppendStream(myMainStream, filtered);
-      }
+//namespace BCG
+//{
+//   public class FullJoinStream : ViewStream
+//   {
+//      private ViewStream myMainStream;
+//      public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
+//         ViewStream v2, string joinCol2, string cmExp2, string empty_col)
+//      {
+//         myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
+//         ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
+//         filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
+//         myMainStream = new AppendStream(myMainStream, filtered);
+//      }
 
-      protected override void myOnFirstSubscriber()
-      {
-         myMainStream.onChange += myHandleMainStreamChange;
-      }
+//      protected override void myOnFirstSubscriber()
+//      {
+//         myMainStream.onChange += myHandleMainStreamChange;
+//      }
 
-      private void myHandleMainStreamChange(object newData)
-      {
-         IBlueRowCollection result = (newData as DataPacket).newRows;
-         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
-      }
-   }
-}
+//      private void myHandleMainStreamChange(object newData)
+//      {
+//         IBlueRowCollection result = (newData as DataPacket).newRows;
+//         myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs

@@ -12,7 +12,7 @@ using System.Collections.Generic;
 using System.Data.SQLite;
 
 
-namespace bts.SilverPine {
+namespace bts.BlueOrb{
 
    public static class SQLiteHelper {
       public static string location = "C:\\Users\\m_tah\\development\\data\\";

src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs

@@ -1,14 +1,12 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using System;
    using System.Data;
    using System.Data.SQLite;
    using System.Diagnostics;
    using System.Collections;
-
-
    using BCG;
 
-   [DataStreamUserRespresentation("SQLite")]
+   [DataStreamUserRespresentation("SQLiteStream")]
    public class SQLiteStream : ViewStream {
       private readonly string _connection_string, _query;
       private Stopwatch _stopwatch = new Stopwatch();
@@ -70,4 +68,5 @@ namespace bts.SilverPine {
 
    }
 
-}
+}
+

src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs

@@ -1,5 +1,6 @@
 using System;
 
+using BCG;
 using BCG.Utilities.General;
 
 using bts;
@@ -87,3 +88,17 @@ namespace BCG {
       }
    }
 }
+
+//namespace bts.BlueOrb {
+//   [DataStreamUserRespresentation("MillisecondStream")]
+//   public class MillisecondStream : ViewStream {
+//      public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
+//         ViewStream rs = new BCG.MillisecondStream(vsd, interval_msecs, size);
+//         mySetRealSource(rs);
+//      }
+
+//      public MillisecondStream(int interval_msecs, int size)
+//         : this(interval_msecs, size) {
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs

@@ -1,124 +1,124 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;
 
-   using bts;
-   using bts.collections;
+//   using bts;
+//   //using bts.collections;
 
-   using System;
-   using System.CodeDom;
-   using System.Collections.Generic;
-   using System.Linq;
+//   using System;
+//   using System.CodeDom;
+//   using System.Collections.Generic;
+//   using System.Linq;
 
-   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
-   // so:
-   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
-   [DataStreamUserRespresentation("ColDiff")]
-   public class ColumnDifferenceStream : ViewStream {
-      private readonly ViewStream _base_stream;
-      private readonly FilterStream.OnFilter _f1, _f2;
-      private readonly string _c1, _c2;
+//   // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
+//   // so:
+//   // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
+//   [DataStreamUserRespresentation("ColDiff")]
+//   public class ColumnDifferenceStream : ViewStream {
+//      private readonly ViewStream _base_stream;
+//      private readonly FilterStream.OnFilter _f1, _f2;
+//      private readonly string _c1, _c2;
 
-      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
-         _base_stream = base_stream;
-         _f1 = f1;
-         _f2 = f2;
-         _c1 = col1;
-         _c2 = col2;
-      }
+//      public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
+//         _base_stream = base_stream;
+//         _f1 = f1;
+//         _f2 = f2;
+//         _c1 = col1;
+//         _c2 = col2;
+//      }
 
-      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
-         FilterStream.OnFilter f1_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
-            return rv.filter(r);
-         };
-         FilterStream.OnFilter f2_del = r => {
-            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
-            return rv.filter(r);
-         };
-         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
-         mySetRealSource(rs);
-      }
+//      public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
+//         FilterStream.OnFilter f1_del = r => {
+//            BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
+//            return rv.filter(r);
+//         };
+//         FilterStream.OnFilter f2_del = r => {
+//            BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
+//            return rv.filter(r);
+//         };
+//         ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
+//         mySetRealSource(rs);
+//      }
 
-      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
-      // we should generlize this for any math operation
-      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
-         // use GlobalMapStream
-         string diff_col_perc_name = $"{diff_col_name}_percentage";
-         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
-            int minuend_idx = Convert.ToInt32(idx);
-            var inp_schema = rows[0].schema;
-            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
-            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
-            // minuend - subtrahend
-            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
-            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
-            int subtrahend_index = minuend_idx + row_offset;
-            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
-            if (subtrahend_index >= 0) {
-               try {
-                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
-                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
-                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
-               } catch (Exception ex) {
-                  Log.assert(false, ex.Message);
-               }
-            }
-            rv.setValue(diff_col_name, diff_val);
-            rv.setValue(diff_col_perc_name, diff_val_perc);
-            return rv;
-         };
-         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
-         if (optional_perc_diff_filter != 0) {
-            rs = new FilterStream(rs, r => {
-               double diff_val_perc = r.get<double>(diff_col_perc_name);
-               double abs_diff_val_perc = Math.Abs(diff_val_perc);
-               return abs_diff_val_perc >= optional_perc_diff_filter;
-            });
-         }
-         mySetRealSource(rs);
-      }
+//      // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
+//      // we should generlize this for any math operation
+//      public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
+//         // use GlobalMapStream
+//         string diff_col_perc_name = $"{diff_col_name}_percentage";
+//         System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
+//            int minuend_idx = Convert.ToInt32(idx);
+//            var inp_schema = rows[0].schema;
+//            var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
+//            out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
+//            // minuend - subtrahend
+//            IBlueRow minuend_row = rows.ElementAt(minuend_idx);
+//            MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
+//            int subtrahend_index = minuend_idx + row_offset;
+//            double diff_val = Double.NaN, diff_val_perc = Double.NaN;
+//            if (subtrahend_index >= 0) {
+//               try {
+//                  IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
+//                  diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
+//                  diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
+//               } catch (Exception ex) {
+//                  Log.assert(false, ex.Message);
+//               }
+//            }
+//            rv.setValue(diff_col_name, diff_val);
+//            rv.setValue(diff_col_perc_name, diff_val_perc);
+//            return rv;
+//         };
+//         ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
+//         if (optional_perc_diff_filter != 0) {
+//            rs = new FilterStream(rs, r => {
+//               double diff_val_perc = r.get<double>(diff_col_perc_name);
+//               double abs_diff_val_perc = Math.Abs(diff_val_perc);
+//               return abs_diff_val_perc >= optional_perc_diff_filter;
+//            });
+//         }
+//         mySetRealSource(rs);
+//      }
 
-      protected override void myOnFirstSubscriber() {
-         _base_stream.onChange += _handle_base_stream_update;
-      }
+//      protected override void myOnFirstSubscriber() {
+//         _base_stream.onChange += _handle_base_stream_update;
+//      }
 
-      protected override void myOnLastSubscriber() {
-         _base_stream.onChange -= _handle_base_stream_update;
-      }
+//      protected override void myOnLastSubscriber() {
+//         _base_stream.onChange -= _handle_base_stream_update;
+//      }
 
-      private void _handle_base_stream_update(object dp_obj) {
-         // could have been done with filter and cartesian prod, and then math
-         DataPacket dp = dp_obj as DataPacket;
-         List<IBlueRow> rows = dp.newRows.to_list();
-         var f1_rows = rows.Where(r => _f1(r));
-         //Log.assert(f1_rows.Count() == 1);
-         var f2_rows = rows.Where(r => _f2(r));
-         //Log.assert(f2_rows.Count() == 1);
-         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
-            IBlueRow f1_r = f1_rows.ElementAt(0);
-            IBlueRow f2_r = f2_rows.ElementAt(0);
-            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         } else {
-            IBlueRow f1_r = rows.ElementAt(0);
-            IBlueRow f2_r = rows.ElementAt(0);
-            double diff = Double.NaN;
-            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
-            BlueTable out_table = new BlueTable(out_schema);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
-            out_row.setValue("diff", diff);
-            out_table.addRow(out_row);
-            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
-            myFireEvent(dp_out);
-         }
+//      private void _handle_base_stream_update(object dp_obj) {
+//         // could have been done with filter and cartesian prod, and then math
+//         DataPacket dp = dp_obj as DataPacket;
+//         List<IBlueRow> rows = dp.newRows.to_list();
+//         var f1_rows = rows.Where(r => _f1(r));
+//         //Log.assert(f1_rows.Count() == 1);
+//         var f2_rows = rows.Where(r => _f2(r));
+//         //Log.assert(f2_rows.Count() == 1);
+//         if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
+//            IBlueRow f1_r = f1_rows.ElementAt(0);
+//            IBlueRow f2_r = f2_rows.ElementAt(0);
+//            double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
+//            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+//            BlueTable out_table = new BlueTable(out_schema);
+//            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+//            out_row.setValue("diff", diff);
+//            out_table.addRow(out_row);
+//            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+//            myFireEvent(dp_out);
+//         } else {
+//            IBlueRow f1_r = rows.ElementAt(0);
+//            IBlueRow f2_r = rows.ElementAt(0);
+//            double diff = Double.NaN;
+//            IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
+//            BlueTable out_table = new BlueTable(out_schema);
+//            MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
+//            out_row.setValue("diff", diff);
+//            out_table.addRow(out_row);
+//            DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
+//            myFireEvent(dp_out);
+//         }
 
-      }
-   }
+//      }
+//   }
 
-}
+//}

src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs

@@ -1,118 +1,118 @@
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using BCG.Utilities.General;
-namespace BCG
-{
-   /// <summary>
-   /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
-   /// The column ordering the 2 tables need not be the same
-   /// </summary>
-   public class AppendStream : UnionStream
-   {
-      private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
-      public AppendStream(ViewStream v1, ViewStream v2)
-      {
-         if (myUseNewAppend)
-            mySetRealSource(new MutableAppendStream(v1, v2));
-         else
-         {
-            myLeftVs = v1;
-            myRightVs = v2;
-         }
-      }
+//using System;
+//using System.Data;
+//using System.Collections;
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+//namespace BCG
+//{
+//   /// <summary>
+//   /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
+//   /// The column ordering the 2 tables need not be the same
+//   /// </summary>
+//   public class AppendStream : UnionStream
+//   {
+//      private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
+//      public AppendStream(ViewStream v1, ViewStream v2)
+//      {
+//         if (myUseNewAppend)
+//            mySetRealSource(new MutableAppendStream(v1, v2));
+//         else
+//         {
+//            myLeftVs = v1;
+//            myRightVs = v2;
+//         }
+//      }
 
-      public AppendStream (params ViewStream[] vss)
-      {
+//      public AppendStream (params ViewStream[] vss)
+//      {
 
-         if (myUseNewAppend)
-            mySetRealSource(new MutableAppendStream(vss));
-         else
-         {
-            if (vss.Length == 1)
-               mySetRealSource(vss[0]);
-            else if (vss.Length > 1) 
-            {
-               AppendStream rv = null;
-               foreach (ViewStream s in vss)
-                  rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
-               mySetRealSource(rv);
-            }
-            else
-               //don't do anything if they didnt' supply any viewstreams
-               mySetRealSource(null);
-         }
+//         if (myUseNewAppend)
+//            mySetRealSource(new MutableAppendStream(vss));
+//         else
+//         {
+//            if (vss.Length == 1)
+//               mySetRealSource(vss[0]);
+//            else if (vss.Length > 1) 
+//            {
+//               AppendStream rv = null;
+//               foreach (ViewStream s in vss)
+//                  rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
+//               mySetRealSource(rv);
+//            }
+//            else
+//               //don't do anything if they didnt' supply any viewstreams
+//               mySetRealSource(null);
+//         }
           
-      }
+//      }
 
-      public override void add (IBlueRowCollection rows, bool isLeft) 
-      {
-         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
-         foreach (IBlueRow row in rows)
-         {
-            UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, 
-               myIdCounter ++, myIdIndex, indexToIndexMap);
-            ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
-            index[row["id"]] =  newRow;
-            ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
-         }
-      }
+//      public override void add (IBlueRowCollection rows, bool isLeft) 
+//      {
+//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+//         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
+//         foreach (IBlueRow row in rows)
+//         {
+//            UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, 
+//               myIdCounter ++, myIdIndex, indexToIndexMap);
+//            ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
+//            index[row["id"]] =  newRow;
+//            ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
+//         }
+//      }
 
-      public override void delete (IBlueRowCollection rows, bool isLeft) 
-      {
-         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-         foreach (IBlueRow axedRow in rows)
-         {
-            int rowId = (int)axedRow["id"];
-            IBlueRow row = (IBlueRow)index[rowId];
-            index.Remove (rowId);
-            ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
-            ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
-         }
-      }
+//      public override void delete (IBlueRowCollection rows, bool isLeft) 
+//      {
+//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+//         foreach (IBlueRow axedRow in rows)
+//         {
+//            int rowId = (int)axedRow["id"];
+//            IBlueRow row = (IBlueRow)index[rowId];
+//            index.Remove (rowId);
+//            ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
+//            ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
+//         }
+//      }
 
-      public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
-      {
-         IDictionary temp = new Hashtable ();
-         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
-         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
-         IBlueRow corrRow;
-         foreach (IBlueRow dRow in dels)
-         {
-            corrRow = index[dRow["id"]] as IBlueRow;
-            (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
-            (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
-            index.Remove (dRow["id"]);
-            if (adds.containsID ((int)dRow["id"]))
-            {
-               IBlueRow r = adds[(int)dRow["id"]];
-               corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
-               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
-               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
-               index.Add (r["id"], corrRow);
-               temp.Add (r["id"], true);
-            }
-         }
-         foreach (IBlueRow aRow in adds)
-         {
-            if (!temp.Contains (aRow["id"]))
-            {
-               corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
-               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
-               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
-               index.Add (aRow["id"], corrRow);
-            }
-         }
-      }
-   }
+//      public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
+//      {
+//         IDictionary temp = new Hashtable ();
+//         IDictionary index = isLeft ? myLeftIndex : myRightIndex;
+//         int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
+//         IBlueRow corrRow;
+//         foreach (IBlueRow dRow in dels)
+//         {
+//            corrRow = index[dRow["id"]] as IBlueRow;
+//            (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
+//            (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
+//            index.Remove (dRow["id"]);
+//            if (adds.containsID ((int)dRow["id"]))
+//            {
+//               IBlueRow r = adds[(int)dRow["id"]];
+//               corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
+//               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
+//               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
+//               index.Add (r["id"], corrRow);
+//               temp.Add (r["id"], true);
+//            }
+//         }
+//         foreach (IBlueRow aRow in adds)
+//         {
+//            if (!temp.Contains (aRow["id"]))
+//            {
+//               corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
+//               (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
+//               (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
+//               index.Add (aRow["id"], corrRow);
+//            }
+//         }
+//      }
+//   }
 
-   public class MergeStream : ViewStream
-   {
-      public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
-      {
-         mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
-      }
-   }
-}
+//   public class MergeStream : ViewStream
+//   {
+//      public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
+//      {
+//         mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs

@@ -2,170 +2,171 @@ using System;
 using System.Collections.Generic;
 
 using mbts;
+//using BCG;
 
 namespace BCG {
-   internal class MutableAppendTable : IBlueRowCollection {
-      private class MyRow : IBlueRow {
-         private IBlueRow myRow;
-         private int myID;
-
-         public MyRow(IBlueRow r, int id) {
-            myRow = r;
-            myID = id;
-         }
-
-         #region IBlueRow Members
-
-         public IBlueTableSchema schema { get { return myRow.schema; } }
-
-         public object this[int index] {
-            get {
-               if (myRow.schema[index].colName == "id")
-                  return myID;
-               else
-                  return myRow[index];
-            }
-         }
-
-         public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
-
-         #endregion
-
-         #region IDictionary Members
-
-         public void Add(object key, object value) { throw new Exception("Row is read only."); }
-
-         public void Clear() { throw new Exception("Row is read only."); }
-
-         public bool Contains(object key) { return myRow.Contains(key); }
-
-         public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public System.Collections.ICollection Keys { get { return myRow.Keys; } }
-
-         public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
-
-         public System.Collections.ICollection Values {
-            get {
-               object[] vals = new object[myRow.Count];
-               CopyTo(vals, 0);
-               return vals;
-            }
-         }
-
-         public object this[object key] {
-            get { return key is string ? this[key as string] : this[(int)key]; }
-            set { throw new Exception("The method or operation is not implemented."); }
-         }
-
-         #endregion
-
-         #region ICollection Members
-
-         public void CopyTo(Array array, int index) {
-            myRow.CopyTo(array, index);
-            array.SetValue(myID, index + myRow.schema["id"].colNo);
-         }
-
-         public int Count { get { return myRow.Count; } }
-
-         public bool IsSynchronized { get { return myRow.IsSynchronized; } }
-
-         public object SyncRoot { get { return myRow.SyncRoot; } }
-
-         #endregion
-
-         #region IEnumerable Members
-
-         System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         #endregion
-      }
-
-      private BlueTable myRows;
-      private int myID = 0;
-      /// <summary>
-      /// maps the table no to a dictionary containing the
-      /// local IDs to a row in the final big table
-      /// </summary>
-      private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
-
-      public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
-
-      public IBlueRow addRow(IBlueRow r, int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
-         IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
-         myRows.addRow(nr);
-         myIDMap[tableNo].Add((int)r["id"], nr);
-         return nr;
-      }
-
-      public IBlueRow removeRow(IBlueRow r, int tableNo) {
-         IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
-         myRows.deleteRow(dr);
-         myIDMap[tableNo].Remove((int)r["id"]);
-         return dr;
-      }
-
-      public IBlueRowCollection removeTable(int tableNo) {
-         if (!myIDMap.ContainsKey(tableNo))
-            return null;
-         else {
-            BlueTable rv = new BlueTable(myRows.schema);
-            foreach (IBlueRow r in myIDMap[tableNo].Values)
-               rv.addRow(r);
-            myRows.deleteRows(rv);
-            myIDMap.Remove(tableNo);
-            return rv;
-         }
-      }
-
-      public void clear() {
-         myRows.clear();
-         myIDMap.Clear();
-      }
-
-      #region IBlueRowCollection Members
-
-      public IBlueRow this[int id] { get { return myRows[id]; } }
-
-      public IBlueTableSchema schema { get { return myRows.schema; } }
-
-      public bool containsID(int id) { return myRows.containsID(id); }
-
-      #endregion
-
-      #region ICollection Members
-
-      public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
-
-      public int Count { get { return myRows.Count; } }
-
-      public bool IsSynchronized { get { return myRows.IsSynchronized; } }
-
-      public object SyncRoot { get { return myRows.SyncRoot; } }
-
-      #endregion
-
-      #region IEnumerable Members
-
-      public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
-
-      #endregion
-   }
-   /// <summary>
+    /// <summary>
    /// MutableStream can append multiple streams
    /// as well as give the user the flexibility
    /// to append (or remove) more streams at
    /// runtime
    /// </summary>
-   [DataStreamUserRespresentation("UnionStream")]
+   //[DataStreamUserRespresentation("UnionStream")]
    public class MutableAppendStream : ViewStream {
+      internal class MutableAppendTable : IBlueRowCollection {
+         private class MyRow : IBlueRow {
+            private IBlueRow myRow;
+            private int myID;
+
+            public MyRow(IBlueRow r, int id) {
+               myRow = r;
+               myID = id;
+            }
+
+            #region IBlueRow Members
+
+            public IBlueTableSchema schema { get { return myRow.schema; } }
+
+            public object this[int index] {
+               get {
+                  if (myRow.schema[index].colName == "id")
+                     return myID;
+                  else
+                     return myRow[index];
+               }
+            }
+
+            public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
+
+            #endregion
+
+            #region IDictionary Members
+
+            public void Add(object key, object value) { throw new Exception("Row is read only."); }
+
+            public void Clear() { throw new Exception("Row is read only."); }
+
+            public bool Contains(object key) { return myRow.Contains(key); }
+
+            public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+            public bool IsFixedSize { get { return true; } }
+
+            public bool IsReadOnly { get { return true; } }
+
+            public System.Collections.ICollection Keys { get { return myRow.Keys; } }
+
+            public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
+
+            public System.Collections.ICollection Values {
+               get {
+                  object[] vals = new object[myRow.Count];
+                  CopyTo(vals, 0);
+                  return vals;
+               }
+            }
+
+            public object this[object key] {
+               get { return key is string ? this[key as string] : this[(int)key]; }
+               set { throw new Exception("The method or operation is not implemented."); }
+            }
+
+            #endregion
+
+            #region ICollection Members
+
+            public void CopyTo(Array array, int index) {
+               myRow.CopyTo(array, index);
+               array.SetValue(myID, index + myRow.schema["id"].colNo);
+            }
+
+            public int Count { get { return myRow.Count; } }
+
+            public bool IsSynchronized { get { return myRow.IsSynchronized; } }
+
+            public object SyncRoot { get { return myRow.SyncRoot; } }
+
+            #endregion
+
+            #region IEnumerable Members
+
+            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+            #endregion
+         }
+
+         private BlueTable myRows;
+         private int myID = 0;
+         /// <summary>
+         /// maps the table no to a dictionary containing the
+         /// local IDs to a row in the final big table
+         /// </summary>
+         private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
+
+         public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
+
+         public IBlueRow addRow(IBlueRow r, int tableNo) {
+            if (!myIDMap.ContainsKey(tableNo))
+               myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
+            IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
+            myRows.addRow(nr);
+            myIDMap[tableNo].Add((int)r["id"], nr);
+            return nr;
+         }
+
+         public IBlueRow removeRow(IBlueRow r, int tableNo) {
+            IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
+            myRows.deleteRow(dr);
+            myIDMap[tableNo].Remove((int)r["id"]);
+            return dr;
+         }
+
+         public IBlueRowCollection removeTable(int tableNo) {
+            if (!myIDMap.ContainsKey(tableNo))
+               return null;
+            else {
+               BlueTable rv = new BlueTable(myRows.schema);
+               foreach (IBlueRow r in myIDMap[tableNo].Values)
+                  rv.addRow(r);
+               myRows.deleteRows(rv);
+               myIDMap.Remove(tableNo);
+               return rv;
+            }
+         }
+
+         public void clear() {
+            myRows.clear();
+            myIDMap.Clear();
+         }
+
+         #region IBlueRowCollection Members
+
+         public IBlueRow this[int id] { get { return myRows[id]; } }
+
+         public IBlueTableSchema schema { get { return myRows.schema; } }
+
+         public bool containsID(int id) { return myRows.containsID(id); }
+
+         #endregion
+
+         #region ICollection Members
+
+         public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
+
+         public int Count { get { return myRows.Count; } }
+
+         public bool IsSynchronized { get { return myRows.IsSynchronized; } }
+
+         public object SyncRoot { get { return myRows.SyncRoot; } }
+
+         #endregion
+
+         #region IEnumerable Members
+
+         public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
+
+         #endregion
+      }
       private bool mySubscriptionActive = false;
       private MutableAppendTable myTable;
       private int myMovingIndex;
@@ -173,7 +174,8 @@ namespace BCG {
       private bool _allign_col_orders = true;
 
       public MutableAppendStream(params ViewStream[] streams)
-         : this(new List<ViewStream>(streams)) { }
+         : this(new List<ViewStream>(streams)) {
+      }
 
       public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
          : this(allign_col_order, new List<ViewStream>(streams)) { }
@@ -256,3 +258,13 @@ namespace BCG {
       }
    }
 }
+
+
+//namespace bts.BlueOrb {
+//   public class UnionStream : ViewStream {
+//      public UnionStream(ViewStream left_stream, ViewStream right_stream) {
+//         ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
+//         mySetRealSource(rs);
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs

@@ -1,63 +1,63 @@
-using System;
+//using System;
 
-namespace BCG
-{
-   /// <summary>
-   /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
-   /// on receiving updates
-   /// </summary>
-   public class UnionOperand 
-   {
-      private bool myIsLeft;
-      private DataPacket myLastArrivedPacket;
-      private UnionStream myMaster;
+//namespace BCG
+//{
+//   /// <summary>
+//   /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
+//   /// on receiving updates
+//   /// </summary>
+//   public class UnionOperand 
+//   {
+//      private bool myIsLeft;
+//      private DataPacket myLastArrivedPacket;
+//      private UnionStream myMaster;
        
-      public UnionOperand (bool isLeft, UnionStream master)
-      {
-         myIsLeft = isLeft;
-         myMaster = master;
-      }
+//      public UnionOperand (bool isLeft, UnionStream master)
+//      {
+//         myIsLeft = isLeft;
+//         myMaster = master;
+//      }
 
-      internal void myChangeHandler (object o)
-      {
-         if (myLastArrivedPacket != null)
-            mySubsequentChangeHandler ((DataPacket)o);
-         else
-         {
-            myLastArrivedPacket = (DataPacket)o;
-            myFirstChangeHandler ((DataPacket)o);
-         }
-      }
+//      internal void myChangeHandler (object o)
+//      {
+//         if (myLastArrivedPacket != null)
+//            mySubsequentChangeHandler ((DataPacket)o);
+//         else
+//         {
+//            myLastArrivedPacket = (DataPacket)o;
+//            myFirstChangeHandler ((DataPacket)o);
+//         }
+//      }
 
-      private void myFirstChangeHandler(DataPacket dp)
-      {
-         myMaster.startSession ();
-         myMaster.add (dp.newRows, myIsLeft);
-         myMaster.endSession ();
-      }
+//      private void myFirstChangeHandler(DataPacket dp)
+//      {
+//         myMaster.startSession ();
+//         myMaster.add (dp.newRows, myIsLeft);
+//         myMaster.endSession ();
+//      }
 
-      private void mySubsequentChangeHandler (DataPacket dp)
-      {
-         myMaster.startSession ();  
-         //myMaster.delete (dp.deletedRows, myIsLeft);
-         //myMaster.add (dp.addedRows, myIsLeft);
-         myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
-         myMaster.endSession ();
-      }
+//      private void mySubsequentChangeHandler (DataPacket dp)
+//      {
+//         myMaster.startSession ();  
+//         //myMaster.delete (dp.deletedRows, myIsLeft);
+//         //myMaster.add (dp.addedRows, myIsLeft);
+//         myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
+//         myMaster.endSession ();
+//      }
 
-      public IBlueTableSchema opSchema
-      {
-         get
-         {
-            if (myLastArrivedPacket == null  || myLastArrivedPacket.newRows == null)
-               return null;
-            return myLastArrivedPacket.newRows.schema;
-         }
-      }
+//      public IBlueTableSchema opSchema
+//      {
+//         get
+//         {
+//            if (myLastArrivedPacket == null  || myLastArrivedPacket.newRows == null)
+//               return null;
+//            return myLastArrivedPacket.newRows.schema;
+//         }
+//      }
 
-      public void init ()
-      {
-         myLastArrivedPacket = null;
-      }
-   }
-}
+//      public void init ()
+//      {
+//         myLastArrivedPacket = null;
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs

@@ -1,119 +1,119 @@
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using BCG.Utilities.General;
-namespace BCG
-{
-   /// <summary>
-   /// Provides a live stream of Union of 2 streams
-   /// </summary>
-   public abstract class UnionStream : ViewStream 
-   {
-      protected UnionOperand myLeftOp, myRightOp;
-      protected ViewStream myLeftVs, myRightVs;
-      protected DataPacket myLastFiredPacket;
-      protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
-      protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
-      protected int myIdCounter, myIdIndex;
+//using System;
+//using System.Data;
+//using System.Collections;
+//using BCG.Utilities;
+//using BCG.Utilities.General;
+//namespace BCG
+//{
+//   /// <summary>
+//   /// Provides a live stream of Union of 2 streams
+//   /// </summary>
+//   public abstract class UnionStream : ViewStream 
+//   {
+//      protected UnionOperand myLeftOp, myRightOp;
+//      protected ViewStream myLeftVs, myRightVs;
+//      protected DataPacket myLastFiredPacket;
+//      protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
+//      protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
+//      protected int myIdCounter, myIdIndex;
 
-      protected override void myOnFirstSubscriber() 
-      {
-         myLeftOp = new UnionOperand (true, this);
-         myRightOp = new UnionOperand (false, this);
-         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
-      }
+//      protected override void myOnFirstSubscriber() 
+//      {
+//         myLeftOp = new UnionOperand (true, this);
+//         myRightOp = new UnionOperand (false, this);
+//         myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+//         myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+//      }
 
-      protected override void myOnLastSubscriber () 
-      {
-         myInit ();
-         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
-         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
-      }
+//      protected override void myOnLastSubscriber () 
+//      {
+//         myInit ();
+//         myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
+//         myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
+//      }
 
-      public void startSession ()
-      {
-         if (myLastFiredPacket == null)
-         {
-            myLastFiredPacket = new DataPacket ();
-            if (myLeftOp.opSchema != null)//ie call back initiated by left operand
-            {
-               myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
-               myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
-               myIdIndex = myLeftOp.opSchema["id"].colNo;
-            }
-            else
-            {
-               myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
-               myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
-               myIdIndex = myRightOp.opSchema["id"].colNo;
-            }
-         }
-         else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
-            myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
-         else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
-            myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-      }
+//      public void startSession ()
+//      {
+//         if (myLastFiredPacket == null)
+//         {
+//            myLastFiredPacket = new DataPacket ();
+//            if (myLeftOp.opSchema != null)//ie call back initiated by left operand
+//            {
+//               myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
+//               myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
+//               myIdIndex = myLeftOp.opSchema["id"].colNo;
+//            }
+//            else
+//            {
+//               myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
+//               myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
+//               myIdIndex = myRightOp.opSchema["id"].colNo;
+//            }
+//         }
+//         else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
+//            myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
+//         else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
+//            myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
+//         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
+//      }
 
-      private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
-      {
-         //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
-         //foreach (string colName in sch1.columnNames)
-         //   BlueEventLog.logAssert (sch2[colName] != null);
-         if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
-            throw new BlueDataException("Union stream: different schemas");
-         foreach(string colName in sch1.columnNames)
-         {
-            if(sch2[colName] == null)
-               throw new BlueDataException("Union stream: can't find " + colName);
-         }
-         int[] retVal = new int[sch1.columnNames.Length];
-         for (int i = 0; i < retVal.Length; i++)
-            retVal[i] = sch2[sch1[i].colName].colNo;
-         return retVal;
-      }
+//      private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
+//      {
+//         //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
+//         //foreach (string colName in sch1.columnNames)
+//         //   BlueEventLog.logAssert (sch2[colName] != null);
+//         if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
+//            throw new BlueDataException("Union stream: different schemas");
+//         foreach(string colName in sch1.columnNames)
+//         {
+//            if(sch2[colName] == null)
+//               throw new BlueDataException("Union stream: can't find " + colName);
+//         }
+//         int[] retVal = new int[sch1.columnNames.Length];
+//         for (int i = 0; i < retVal.Length; i++)
+//            retVal[i] = sch2[sch1[i].colName].colNo;
+//         return retVal;
+//      }
 
-      public virtual void endSession ()
-      {
-         myFireEvent (myLastFiredPacket.Clone ());
-      }
+//      public virtual void endSession ()
+//      {
+//         myFireEvent (myLastFiredPacket.Clone ());
+//      }
 
-      public abstract void add (IBlueRowCollection rows, bool isLeft);
+//      public abstract void add (IBlueRowCollection rows, bool isLeft);
 
-      public abstract void delete (IBlueRowCollection rows, bool isLeft);
+//      public abstract void delete (IBlueRowCollection rows, bool isLeft);
 
-      public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
+//      public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
       
-      private void myInit ()
-      {
-         myLastFiredPacket = null;
-         myLeftOp.init ();
-         myRightOp.init ();
-         myLeftIndex.Clear ();
-         myRightIndex.Clear ();
-         myIdCounter = myIdIndex = 0;
-         myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
-      }
+//      private void myInit ()
+//      {
+//         myLastFiredPacket = null;
+//         myLeftOp.init ();
+//         myRightOp.init ();
+//         myLeftIndex.Clear ();
+//         myRightIndex.Clear ();
+//         myIdCounter = myIdIndex = 0;
+//         myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
+//      }
 
-      /// <summary>
-      /// Used for test purposes only. Users of this class specifically, should ignore it 
-      /// (too bad, the NUnit cant see the private methods)
-      /// </summary>
-      /// <param name="dp"></param>
-      /// <param name="isLeft"></param>
-      internal void receiveStream (DataPacket dp, bool isLeft)
-      {
-         if (isLeft)
-            myLeftOp.myChangeHandler (dp);
-         else
-            myRightOp.myChangeHandler (dp);
-      }
-      public void receiveStreamTest(DataPacket dp, bool isLeft)
-      {
-          receiveStream(dp, isLeft);
-      }
-   }
-}
+//      /// <summary>
+//      /// Used for test purposes only. Users of this class specifically, should ignore it 
+//      /// (too bad, the NUnit cant see the private methods)
+//      /// </summary>
+//      /// <param name="dp"></param>
+//      /// <param name="isLeft"></param>
+//      internal void receiveStream (DataPacket dp, bool isLeft)
+//      {
+//         if (isLeft)
+//            myLeftOp.myChangeHandler (dp);
+//         else
+//            myRightOp.myChangeHandler (dp);
+//      }
+//      public void receiveStreamTest(DataPacket dp, bool isLeft)
+//      {
+//          receiveStream(dp, isLeft);
+//      }
+//   }
+//}

src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs

@@ -0,0 +1,73 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts.api.core;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+
+public class TableUpdateDataPacket<T> : IDataPacket<T> where T : IBlueRow {
+   readonly TableUpdate<T, int> table;
+   readonly IBlueTableSchema schema;
+
+   public TableUpdateDataPacket(TableUpdate<T, int> table, IBlueTableSchema schema) {
+      this.table = table;
+      this.schema = schema;
+   }
+
+   public IBlueRowCollection<T> newRows {
+      get {
+         var rv = new BlueTableImmutable<T>(schema);
+         return rv;
+      }
+   }
+
+   public IBlueRowCollection<T> addedRows => throw new NotImplementedException();
+
+   public IBlueRowCollection<T> deletedRows => throw new NotImplementedException();
+
+   class MyRowCollection : IBlueRowCollection<T> {
+      readonly TableUpdate<T, int> table;
+      readonly IEnumerable<T> items;
+      Lazy<SortedList<int, T>> itemList;
+
+      public MyRowCollection(IEnumerable<T> items, IBlueTableSchema schema) {
+         this.items = items;
+         this.schema = schema;
+         itemList = new(() => {
+            var rv = new SortedList<int, T>();
+            foreach (var item in this.items) {
+               rv.Add(item.Id(), item);
+            }
+            return rv;
+         });
+      }
+
+      public int Count => itemList.Value.Count;
+      public bool IsSynchronized => false;
+      public object SyncRoot => this;
+
+      public T this[int id] => itemList.Value[id];
+
+      public IBlueTableSchema schema { get; }
+
+      public bool containsID(int id) =>
+         itemList.Value.ContainsKey(id);
+
+      public void CopyTo(Array array, int index) {
+      }
+
+      public IEnumerator<T> GetEnumerator() {
+         throw new NotImplementedException();
+      }
+
+      IEnumerator IEnumerable.GetEnumerator() {
+         throw new NotImplementedException();
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -114,6 +114,15 @@ namespace BCG {
             rv.Add(r.get<T>(col));
          return rv;
       }
+      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, int start_idx, int n_items,string col) {
+         List<T> rv = new List<T>();
+         int end_idx = Math.Min(start_idx + n_items, rows.Count());
+         for (int i = start_idx; i < end_idx; i++) {
+            var r = rows.ElementAt(i);
+            rv.Add(r.get<T>(col));
+         }
+         return rv;
+      }
       public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
          List<T> rv = new List<T>();
          int i = 0;

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs

@@ -0,0 +1,56 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+public class BlueRowCollection : IBlueRowCollection {
+   readonly ICollection source;
+   readonly Func<int, IBlueRow> findRow;
+   readonly Func<int, bool> containsKey;
+
+   public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
+      this.source = source;
+      this.schema = schema;
+      if (findRow is null || containsKey is null) {
+         if (source is IBlueRowCollection brc) {
+            findRow ??= id => brc[id];
+            containsKey ??= brc.containsID;
+         } else if (source is IDictionary dict) {
+            findRow ??= id => dict[id] as IBlueRow ?? throw new KeyNotFoundException();
+            containsKey ??= id => dict.Contains(id);
+         } else if (source is IDictionary<int, IBlueRow> genDict) {
+            findRow ??= id => genDict[id];
+            containsKey ??= genDict.ContainsKey;
+         } else if (source is IReadOnlyDictionary<int, IBlueRow> roGenDict) {
+            findRow ??= id => roGenDict[id];
+            containsKey = roGenDict.ContainsKey;
+         } else {
+            var cachedRows = new Lazy<Dictionary<int, IBlueRow>>(() =>
+               source.Cast<IBlueRow>().ToDictionary(r => r.Id()));
+            findRow ??= id => cachedRows.Value[id];
+            // Don't remove this lambda expression. Intellisense may suggest it, but that would
+            // instantiate 'cachedRows' immediately and defeat the purpose of using Lazy<T>.
+            containsKey ??= id => cachedRows.Value.ContainsKey(id);
+         }
+      }
+      this.findRow = findRow;
+      this.containsKey = containsKey;
+   }
+
+   public IBlueRow this[int id] => findRow(id);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      containsKey(id);
+
+   public IEnumerator GetEnumerator() =>
+      source.GetEnumerator();
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs

@@ -96,3 +96,125 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
          new(schema, table.ToImmutable());
    }
 }
+
+public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly ImmutableDictionary<int, T> rows;
+
+   public BlueTableImmutable(IBlueTableSchema schema)
+      : this(schema, ImmutableDictionary<int, T>.Empty) { }
+
+   public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
+      this.schema = schema;
+      this.rows = rows;
+   }
+
+   public int Count => rows.Count;
+   public bool IsSynchronized => false;
+   public IBlueTableSchema schema { get; }
+   public object SyncRoot => this;
+
+   public T this[int id] => rows[id];
+
+   public bool containsID(int id) =>
+      rows.ContainsKey(id);
+
+   public void CopyTo(Array array, int index) {
+      foreach (var r in rows.Values) {
+         array.SetValue(r, index++);
+      }
+   }
+
+   public IEnumerator<T> GetEnumerator() =>
+      rows.Values.GetEnumerator();
+
+   IEnumerator IEnumerable.GetEnumerator() =>
+      GetEnumerator();
+
+   public DataPacketBuilder ToDataPacketBuilder() =>
+      new(this);
+
+   public class DataPacketBuilder {
+      readonly IBlueTableSchema schema;
+      readonly ImmutableDictionary<int, T>.Builder table;
+      readonly ImmutableDictionary<int, T>.Builder added =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+      readonly ImmutableDictionary<int, T>.Builder removed =
+         ImmutableDictionary<int, T>.Empty.ToBuilder();
+
+      public DataPacketBuilder(BlueTableImmutable<T> table) {
+         schema = table.schema;
+         this.table = table.rows.ToBuilder();
+      }
+
+      public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
+         var id = row.Id();
+         if (table.TryGetValue(id, out var orig)) {
+            if (comparer?.Equals(row, orig) ?? false) {
+               // No op: This row was already in the table (according to 'comparer').
+               // Remove it from 'added' and 'removed'.
+               added.Remove(id);
+               removed.Remove(id);
+               return;
+            }
+            // Change op: A row with this id was already in the table,
+            // but its values have changed (according to 'comparer').
+            removed[id] = orig;
+         }
+         added[id] = row;
+      }
+
+      public void Remove(int id) {
+         added.Remove(id);
+         if (table.TryGetValue(id, out var orig)) {
+            removed[id] = orig;
+         }
+      }
+
+      public DataPacket ToDataPacket(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new(newRows, addedRows, removedRows);
+      }
+
+      public IDataPacket<T> ToDataPacketT(bool reset = true) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
+      public IDataPacket<T> ToDataPacketT(
+         out BlueTableImmutable<T> newRows,
+         out BlueTableImmutable<T> addedRows,
+         out BlueTableImmutable<T> removedRows,
+         bool reset = true
+      ) {
+         table.RemoveRange(removed.Keys);
+         table.AddRange(added);
+         newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
+         addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
+         removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
+         if (reset) {
+            added.Clear();
+            removed.Clear();
+         }
+         return new DataPacket<T>(newRows, addedRows, removedRows);
+      }
+
+      public BlueTableImmutable<T> ToImmutable() =>
+         new(schema, table.ToImmutable());
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs

@@ -0,0 +1,29 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+public class BlueTableSchemaBuilder {
+   readonly string? tableName;
+   readonly List<(string Name, Type Type)> colDefs = new();
+
+   public BlueTableSchemaBuilder(string? tableName = null) =>
+      this.tableName = tableName;
+
+   public void Add<T>(string Name) =>
+      colDefs.Add((Name, typeof(T)));
+
+   public void Add(string Name, Type Type) =>
+      colDefs.Add((Name, Type));
+
+   public IBlueTableSchema ToSchema(string? tableName = null) {
+      var colInfos = new ColumnInfo[colDefs.Count];
+      for (int i = 0; i < colInfos.Length; i++) {
+         colInfos[i] = new(colDefs[i].Name, i, colDefs[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName ?? this.tableName);
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaFactory.cs

@@ -0,0 +1,16 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+
+public class BlueTableSchemaFactory {
+   public static IBlueTableSchema Create(string tableName, params (string Name, Type Type)[] columns) {
+      var colInfos = new ColumnInfo[columns.Length];
+      for (int i = 0; i < columns.Length; i++) {
+         colInfos[i] = new(columns[i].Name, i, columns[i].Type);
+      }
+      return new BlueTableSchema(colInfos, tableName);
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs

@@ -0,0 +1,39 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+public static class BlueTableSelectorWrapper {
+   public static BlueTableSelectorWrapper<T> Create<T>(
+      IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) where T : IBlueRow =>
+      new(source, selector, schema);
+}
+
+public class BlueTableSelectorWrapper<T> : IBlueRowCollection<T> where T : IBlueRow {
+   readonly IBlueRowCollection source;
+   readonly Func<IBlueRow, T> selector;
+
+   public BlueTableSelectorWrapper(IBlueRowCollection source, Func<IBlueRow, T> selector, IBlueTableSchema? schema = null) {
+      this.source = source;
+      this.selector = selector;
+      this.schema = schema ?? source.schema;
+   }
+
+   public T this[int id] => selector(source[id]);
+
+   public IBlueTableSchema schema { get; }
+
+   public int Count => source.Count;
+
+   public bool containsID(int id) =>
+      source.containsID(id);
+
+   public IEnumerator<T> GetEnumerator() {
+      foreach (IBlueRow r in source) {
+         yield return selector(r);
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Text.Json.Serialization;
 
 namespace BCG {
    /// <summary>
@@ -56,9 +57,16 @@ namespace BCG {
       private Type myColType;
       private object myColValue;
 
+      [JsonPropertyName("colNo")]
       public int colNo { get { return myColNo; } }
+
+      [JsonPropertyName("colName")]
       public string colName { get { return myColName; } }
+
+      [JsonPropertyName("colType")]
       public Type colType { get { return myColType; } }
+
+      [JsonIgnore]
       public object colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
 
 namespace BCG
 {
@@ -15,5 +16,23 @@ namespace BCG
       IBlueRow this[int id]{get;}
       IBlueTableSchema schema {get;}
       bool containsID (int id);
+
+      bool ICollection.IsSynchronized => false;
+      object ICollection.SyncRoot => this;
+
+      void ICollection.CopyTo(Array array, int index) {
+         foreach (IBlueRow r in this) {
+            array.SetValue(r, index++);
+         }
+      }
+   }
+
+   public interface IBlueRowCollection<T> : IBlueRowCollection, IReadOnlyCollection<T> where T : IBlueRow {
+      new T this[int id] { get; }
+
+      IBlueRow IBlueRowCollection.this[int id] => this[id];
+
+      IEnumerator IEnumerable.GetEnumerator() =>
+         GetEnumerator();
    }
 }

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -3,6 +3,8 @@ using System.Collections;
 using System.Data;
 using BCG.Utilities.General;
 
+using Bts.DataStreamCore;
+
 namespace BCG
 {
 	[Serializable]
@@ -183,6 +185,7 @@ namespace BCG
             return retVal;
          }
       }
+      IBlueTableSchema IDataPacket.Schema => schema;
 
       /*          
        status  D total  A total  D this   A This   D' total A' Total 
@@ -314,6 +317,24 @@ namespace BCG
       }
    }
 
+   public class DataPacket<T> : IDataPacket<T> where T : IBlueRow {
+      public DataPacket(IBlueRowCollection<T> newRows, IBlueRowCollection<T> addedRows, IBlueRowCollection<T> deletedRows) {
+         this.newRows = newRows;
+         this.addedRows = addedRows;
+         this.deletedRows = deletedRows;
+      }
+
+      public IBlueRowCollection<T> newRows { get; }
+      public IBlueRowCollection<T> addedRows { get; }
+      public IBlueRowCollection<T> deletedRows { get; }
+
+      public static DataPacket<T> Empty(IBlueTableSchema schema) =>
+         new(
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema),
+            new BlueTableImmutable<T>(schema));
+   }
+
    public static class DataPacketExtensions {
       public static bool HasChanges(this DataPacket dataPacket) =>
          dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;

src/DataStreamSuite/DataStreamCore/IDataPacket.cs

@@ -9,7 +9,22 @@ namespace BCG
 {
    public interface IDataPacket
    {
+      IBlueTableSchema Schema { get; }
       //this is an ICollection of IDictionary
       void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
    }
+
+   public interface IDataPacket<T> : IDataPacket where T : IBlueRow {
+      IBlueRowCollection<T> newRows { get; }
+      IBlueRowCollection<T> addedRows { get; }
+      IBlueRowCollection<T> deletedRows { get; }
+
+      IBlueTableSchema IDataPacket.Schema => newRows.schema;
+
+      void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
+         new_rows = newRows;
+         added_rows = addedRows;
+         deleted_rows = deletedRows;
+      }
+   }
 }

src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs

@@ -147,7 +147,7 @@ namespace bts.BlueOrb {
       }
    }
 
-   [DataStreamUserRespresentation("FutureHistoricalBars")]
+   [DataStreamUserRespresentation("FutureHistoricalBarsStream")]
    public class FutureHistoricalBarsStream : ViewStream {
       public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
          DateTime dt_range_beg = DateTime.Parse(range_beg);

src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs

@@ -1,8 +1,8 @@
 using bts.SilverPine;
 
-namespace bts.SilverPine {
-   using bts.api;
-   using bts.spark;
+namespace bts.BlueOrb {
+   //using bts.api;
+   //using bts.spark;
 
    using System;
 
@@ -16,7 +16,7 @@ namespace bts.SilverPine {
          public string contract_text;
       }
       public class FutureSpreadContract {
-         public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
+         public List<Tuple<double, ContractUtils.FutureTradableContract>> legs;
          public string symbol {
             get {
                return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
@@ -206,7 +206,7 @@ namespace bts.SilverPine {
          }
          // tehre are some non calendar spreads we cant parse
          //Log.assert(false);
-         return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
+         return ContractUtils.ContractType.FutureSpread;
       }
 
       public static ContractCategory get_category(string symbol) {

src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs

@@ -1,23 +1,23 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.BlueOrb {
+//   using BCG;
 
-   // Finds the regions around specified events 
-   [DataStreamUserRespresentation("EventsPriceAction")]
-   public class EventsPriceActionStream : ViewStream {
-      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
-            string post_event_period, int n_events) :
-         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
-         n_events, ViewStreamDatabase.instance) { }
+//   // Finds the regions around specified events 
+//   [DataStreamUserRespresentation("EventsPriceAction")]
+//   public class EventsPriceActionStream : ViewStream {
+//      public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
+//            string post_event_period, int n_events) :
+//         this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
+//         n_events, ViewStreamDatabase.instance) { }
 
-      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
-      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
-         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
-         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
-         ViewStream events_str = new EventsStream(vsd, event_type);
-         // todo
-         //events_str = new TopStream(events_str, n_events, "-ts");
-         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
-         mySetRealSource(rs);
-      }
-   }
-}
+//      public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
+//      TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
+//         // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
+//         FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
+//         ViewStream events_str = new EventsStream(vsd, event_type);
+//         // todo
+//         //events_str = new TopStream(events_str, n_events, "-ts");
+//         ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
+//         mySetRealSource(rs);
+//      }
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs

@@ -1,20 +1,20 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;
 
-   public class IndicatorStream : ViewStream {
-      enum Indicator {
-         StandardDeviation = 0
-      }
-      public class IndicatorInfo {
-         Indicator indicator;
-         IDictionary<string, object> parameters;
-      }
-      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
+//   public class IndicatorStream : ViewStream {
+//      enum Indicator {
+//         StandardDeviation = 0
+//      }
+//      public class IndicatorInfo {
+//         Indicator indicator;
+//         IDictionary<string, object> parameters;
+//      }
+//      public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
 
-      }
-      public IndicatorStream(string symbol_name, IndicatorInfo info) {
+//      }
+//      public IndicatorStream(string symbol_name, IndicatorInfo info) {
 
-      }
+//      }
 
-   }
-}
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs

@@ -1,57 +1,57 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;
 
-   using ZstdSharp.Unsafe;
+//   using ZstdSharp.Unsafe;
 
-   [DataStreamUserRespresentation("AbsVolume")]
-   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
-   // but with some thought can perhaps be extended)
-   public class AbsoluteVolumeStream : ViewStream {
-      // TODO, abstract this out
-      // adds contract_type to OHLCV stream
-      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
-         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
-         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
-         rs = new SuperMapStream(rs,
-         s => create_schema(s),
-         r => {
-            IBlueTableSchema out_schema = create_schema(r.schema);
-            string contract_txt = r.get<string>("contract");
-            var contr_type = ContractUtils.get_type(contract_txt);
-            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
-            out_row.setValue("contract_type", contr_type.ToString());
-            return new IBlueRow[] { out_row };
-         });
-         return rs;
-      }
+//   [DataStreamUserRespresentation("AbsVolume")]
+//   // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
+//   // but with some thought can perhaps be extended)
+//   public class AbsoluteVolumeStream : ViewStream {
+//      // TODO, abstract this out
+//      // adds contract_type to OHLCV stream
+//      private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
+//         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+//         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+//         Func<IBlueTableSchema, IBlueTableSchema> create_schema =
+//         s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
+//         rs = new SuperMapStream(rs,
+//         s => create_schema(s),
+//         r => {
+//            IBlueTableSchema out_schema = create_schema(r.schema);
+//            string contract_txt = r.get<string>("contract");
+//            var contr_type = ContractUtils.get_type(contract_txt);
+//            MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
+//            out_row.setValue("contract_type", contr_type.ToString());
+//            return new IBlueRow[] { out_row };
+//         });
+//         return rs;
+//      }
 
-      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
-      : this(symbol, ctype, d1, d1) { }
+//      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
+//      : this(symbol, ctype, d1, d1) { }
 
-      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
-      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
-      }
+//      public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
+//      : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
+//      }
 
-      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
-         if (dt1 < dt2) {
-            DateTime aux = dt1;
-            dt1 = dt2;
-            dt2 = aux;
-         }
-      }
+//      static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
+//         if (dt1 < dt2) {
+//            DateTime aux = dt1;
+//            dt1 = dt2;
+//            dt2 = aux;
+//         }
+//      }
 
-      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
-         swap_if_necessary(ref d1, ref d2);
-         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
-         ohlcv_str = new FilterStream(ohlcv_str,
-            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
-         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
-         ViewStream rs = new MaxStream(ohlcv_str, "volume");
-         rs = new NewProjectStream(rs, "volume:total_volume contract !");
-         mySetRealSource(rs);
-      }
+//      private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
+//         swap_if_necessary(ref d1, ref d2);
+//         ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
+//         ohlcv_str = new FilterStream(ohlcv_str,
+//            r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
+//         ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
+//         ViewStream rs = new MaxStream(ohlcv_str, "volume");
+//         rs = new NewProjectStream(rs, "volume:total_volume contract !");
+//         mySetRealSource(rs);
+//      }
 
-   }
-}
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG;
 
    using SharpCompress.Common.Rar;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG;
 
    using bts.api;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG;
 
    using System.Diagnostics;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine{
+namespace bts.BlueOrb{
    using BCG;
    public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
       private IBlueTableSchema _schema;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG;
 
    using MathNet.Numerics.LinearAlgebra.Factorization;

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using MathNet.Numerics.LinearRegression;
 
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
 
    public static class SpreadWeightCalculator {
       /// <summary>

src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/StandardDeviationStream.cs

@@ -1,40 +1,40 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;
 
-   using MathNet.Numerics.Statistics;
+//   using MathNet.Numerics.Statistics;
 
-   // calculates the reazlied standard deviation of a series of prices
-   public class StandardDeviationStream : ViewStream {
-      public StandardDeviationStream(string symbol, string std_dev_period)
-      :
-      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
+//   // calculates the reazlied standard deviation of a series of prices
+//   public class StandardDeviationStream : ViewStream {
+//      public StandardDeviationStream(string symbol, string std_dev_period)
+//      :
+//      this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
 
-      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
-         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
-         int n_days = std_dev_period.Days;
-         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
-         IBlueTableSchema out_schema = null;
-         int id = 0;
-         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
-            int iidx = Convert.ToInt32(idx);
-            if (rows == null || rows.Count == 0)
-               return null;
-            IBlueRow ir = rows.ElementAt(iidx);
-            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
-            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
-            rv.setValue("std_dev", Double.NaN);
-            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
-            if (last_n_close_prices.Count() >= n_days) {
-               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
-               rv.setValue("std_dev", std_dev);
-            }
-            //rv.setValue("id", id++);
-            return rv;
-         };
-         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
-         rs = new NullFilterStream(rs, "std_dev", true);
-         mySetRealSource(rs);
-      }
+//      public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
+//         // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
+//         int n_days = std_dev_period.Days;
+//         ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
+//         IBlueTableSchema out_schema = null;
+//         int id = 0;
+//         Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
+//            int iidx = Convert.ToInt32(idx);
+//            if (rows == null || rows.Count == 0)
+//               return null;
+//            IBlueRow ir = rows.ElementAt(iidx);
+//            out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
+//            MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
+//            rv.setValue("std_dev", Double.NaN);
+//            IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
+//            if (last_n_close_prices.Count() >= n_days) {
+//               double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
+//               rv.setValue("std_dev", std_dev);
+//            }
+//            //rv.setValue("id", id++);
+//            return rv;
+//         };
+//         ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
+//         rs = new NullFilterStream(rs, "std_dev", true);
+//         mySetRealSource(rs);
+//      }
 
-   }
-}
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG.Utilities.General;
 
    using bts.md;

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -1,5 +1,6 @@
-namespace BCG {
+namespace bts.BlueOrb{
    using BCG.Utilities.General;
+   using BCG;
 
    using bts.md;
 

src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG;
 
    [DataStreamUserRespresentation("Events")]
@@ -15,4 +15,4 @@ namespace bts.SilverPine {
          mySetRealSource(rs);
       }
    }
-}
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb{
    using BCG;
 
    using bts;

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using BCG.Utilities.General;
 
    using bts.api;

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs

@@ -1,37 +1,37 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;
 
-   [DataStreamUserRespresentation("OHLCVPriceDiff")]
-   public class OHLCVPriceDiffStream : ViewStream {
-      // TODO, abstract this out
-      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
-         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
-         var ohlcv_params = OHLCVStream.get_params(d1, d2);
-         var cont_cat = tup.Item1;
-         string symbol = tup.Item2;
-         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
-         switch (cont_cat) {
-            case ContractUtils.ContractCategory.FutureString:
-            case ContractUtils.ContractCategory.SpreadString:
-               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
-               break;
-            case ContractUtils.ContractCategory.Symbol:
-               rs = new FuturesActiveContractStream(rs);
-               break;
-         }
-         return rs;
-      }
+//   [DataStreamUserRespresentation("OHLCVPriceDiff")]
+//   public class OHLCVPriceDiffStream : ViewStream {
+//      // TODO, abstract this out
+//      private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
+//         var tup = ContractUtils.get_symbol_category(symbol_or_contract);
+//         var ohlcv_params = OHLCVStream.get_params(d1, d2);
+//         var cont_cat = tup.Item1;
+//         string symbol = tup.Item2;
+//         ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
+//         switch (cont_cat) {
+//            case ContractUtils.ContractCategory.FutureString:
+//            case ContractUtils.ContractCategory.SpreadString:
+//               rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
+//               break;
+//            case ContractUtils.ContractCategory.Symbol:
+//               rs = new FuturesActiveContractStream(rs);
+//               break;
+//         }
+//         return rs;
+//      }
 
-      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
-      : this(symbol_or_contract, d1, d2, "close") { }
+//      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
+//      : this(symbol_or_contract, d1, d2, "close") { }
 
-      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
-         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
-         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
-                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
-         rs = new NewProjectStream(rs, "diff !");
-         mySetRealSource(rs);
-      }
+//      public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
+//         ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
+//         rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
+//                                              r => d2.Equals(r.get_datetime("ts_event")), col, col);
+//         rs = new NewProjectStream(rs, "diff !");
+//         mySetRealSource(rs);
+//      }
 
-   }
-}
+//   }
+//}

src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs

@@ -1,4 +1,4 @@
-namespace bts.SilverPine {
+namespace bts.BlueOrb {
    using bts;
    using BCG;
    using MongoDB.Driver;

src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs

@@ -1,8 +1,7 @@
 using System;
 using BCG;
 
-namespace bts.SilverPine
-{
+namespace bts.BlueOrb {
 /*
    ----------------------------------------------------------------------
                                *Expression*

src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs

@@ -1,32 +1,32 @@
-namespace bts.SilverPine {
-   using BCG;
+//namespace bts.SilverPine {
+//   using BCG;
 
-   // this is an internal stream. its job is to take an OHLCV stream
-   // and filter it for 
-   // 1. symbol (table_name and active contract) and 
-   // 2. time  (can be a day of the week, month, and a time of the day period)
-   public class BarsSearcherStream : ViewStream {
-      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
-                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
-         ViewStream rs = new OHLCVStream(symbol, bar_size);
-         rs = new DateTimeExpanstionStream(rs, "ts_event");
-         rs = new FilterStream(rs, r => {
-            DateTime ts = r.get_datetime("ts_event");
-            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
-                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
-         });
-         // look at symbol and either filter ir ot do the active contracts
-         // assuming its symbol
-         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
-                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
-         // filter it for the active contract only
-         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
-         mySetRealSource(rs);
-      }
-      public BarsSearcherStream(string symbol, string bar_size,
-                                 string day_of_week, string tod_begin, string tod_end)
-                                 :
-                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
-                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
-   }
-}
+//   // this is an internal stream. its job is to take an OHLCV stream
+//   // and filter it for 
+//   // 1. symbol (table_name and active contract) and 
+//   // 2. time  (can be a day of the week, month, and a time of the day period)
+//   public class BarsSearcherStream : ViewStream {
+//      public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
+//                                 string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
+//         ViewStream rs = new OHLCVStream(symbol, bar_size);
+//         rs = new DateTimeExpanstionStream(rs, "ts_event");
+//         rs = new FilterStream(rs, r => {
+//            DateTime ts = r.get_datetime("ts_event");
+//            return r.get<string>("ts_event_day") == day_of_week.ToString() &&
+//                  ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
+//         });
+//         // look at symbol and either filter ir ot do the active contracts
+//         // assuming its symbol
+//         rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
+//                                 new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
+//         // filter it for the active contract only
+//         rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
+//         mySetRealSource(rs);
+//      }
+//      public BarsSearcherStream(string symbol, string bar_size,
+//                                 string day_of_week, string tod_begin, string tod_end)
+//                                 :
+//                                 this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
+//                                       TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
+//   }
+//}

src/DataStreamSuite/SharedStreams/NL/NLStream.cs

@@ -2,34 +2,38 @@ namespace bts.SilverPine {
    using BCG;
 
    using bts;
-   using bts.SilverPine;
+   //using bts.SilverPine;
 
-   [DataStreamUserRespresentation("nl")]
-   public class NLStream : ViewStream {
-      private readonly string _nl_prompt;
-      private ViewStream _nl_to_dsl_stream, _dsl_stream;
-      public NLStream(string nl_prompt) {
-         _nl_prompt = nl_prompt;
-      }
+   namespace bts.BlueOrb {
 
-      protected override void myOnFirstSubscriber() {
-         _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
-         _nl_to_dsl_stream.onChange += _handle_dsl_callback;
-      }
+      [DataStreamUserRespresentation("nl")]
+      public class NLStream : ViewStream {
+         private readonly string _nl_prompt;
+         private ViewStream _nl_to_dsl_stream, _dsl_stream;
+         public NLStream(string nl_prompt) {
+            _nl_prompt = nl_prompt;
+         }
 
-      private void _handle_dsl_callback(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         var r = dp.newRows.to_list().ElementAt(0);
-         string dsl = r.get<string>("response");
-         Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
-         _dsl_stream = new DSLStream(dsl);
-         _dsl_stream.onChange += _handle_data_cb;
-      }
+         protected override void myOnFirstSubscriber() {
+            _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
+            _nl_to_dsl_stream.onChange += _handle_dsl_callback;
+         }
 
-      private void _handle_data_cb(object dp_obj) {
-         var dp = dp_obj as DataPacket;
-         DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
-         myFireEvent(dp_out);
+         private void _handle_dsl_callback(object dp_obj) {
+            DataPacket dp = dp_obj as DataPacket;
+            var r = dp.newRows.to_list().ElementAt(0);
+            string dsl = r.get<string>("response");
+            Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
+            _dsl_stream = new DSLStream(dsl);
+            _dsl_stream.onChange += _handle_data_cb;
+         }
+
+         private void _handle_data_cb(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+         }
       }
    }
-}
+
+}

src/app_sparkd/CMakeLists.txt

@@ -122,7 +122,7 @@ sym_link_python_module(
 target_link_libraries(
   sparkd
   spark
-  binance_execution
+  binance
   okx_execution
   python_interop
   python_interop_module

src/app_sparkd/Dockerfile

@@ -7,6 +7,7 @@ ARG PYTHON_REQUIREMENTS="requirements.txt"
 FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
 ARG CONFIG_DIR
 ARG SOURCE_DIR
+ARG PYTHON_REQUIREMENTS
 
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev

src/app_sparkd/configurations/binance_dev/.env

@@ -1,2 +1,3 @@
 SPARKD__CONFIG_DIR=src/app_sparkd/configurations/binance_dev
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/binance_dev/requirements.txt
 SPARKD__SHM_SIZE=8gb

src/app_sparkd/configurations/binance_dev/requirements.txt

@@ -0,0 +1,11 @@
+certifi==2025.4.26
+charset-normalizer==3.4.2
+dotenv==0.9.9
+idna==3.10
+numpy==2.2.6
+pybit==5.11.0
+pycryptodome==3.23.0
+python-dotenv==1.1.0
+requests==2.32.4
+urllib3==2.4.0
+websocket-client==1.8.0

src/app_sparkd/configurations/bybit_algo_test/.env

@@ -0,0 +1,6 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/bybit_algo_test
+SPARKD__RUNTIME_CONF_DIR=/home/bts/8000/sparkdata/conf
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/alphaworx_zen_hk_live-preview/requirements.txt
+SPARKD__SHM_SIZE=8gb
+SPARKD__MEMORY=8gb
+SPARKD__IMAGE=sparkd_symbols

src/app_sparkd/configurations/bybit_algo_test/py_bybit_md_provider.json

@@ -0,0 +1,18 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "MOVE",
+    "DUCK",
+    "ETH",
+    "BTC",
+    "SOL",
+    "KAITO"
+  ],
+  "use_testnet": false,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}

src/app_sparkd/configurations/bybit_algo_test/requirements.txt

@@ -0,0 +1,10 @@
+certifi==2025.4.26
+charset-normalizer==3.4.2
+dotenv==0.9.9
+idna==3.10
+pybit==5.11.0
+pycryptodome==3.23.0
+python-dotenv==1.1.0
+requests==2.32.4
+urllib3==2.4.0
+websocket-client==1.8.0

src/app_sparkd/configurations/bybit_algo_test/spark.conf

@@ -0,0 +1,58 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "bybit_1"
+      venue: VENUE_BYBIT
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_interop_plugins"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+    }
+  }
+}
+
+SoundNotifications {
+  value {
+    enabled: true
+  }
+}
+

src/app_sparkd/configurations/bybit_algo_test/symbols.conf

@@ -0,0 +1,32 @@
+SymbolLists {
+  value {
+    list {
+      key: "BybitSpots"
+      symbol: "MOVE"
+      symbol: "DUCK"
+      symbol: "ETH"
+      symbol: "BTC"
+      symbol: "SOL"
+      symbol: "KAITO"
+    }
+    list {
+      key: "BybitFutures"
+      symbol: "MOVE"
+      symbol: "DUCK"
+      symbol: "ETH"
+      symbol: "BTC"
+      symbol: "SOL"
+      symbol: "KAITO"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BYBIT"
+      spot: @SymbolLists.list[BybitSpots].symbol
+      future: @SymbolLists.list[BybitFutures].symbol
+    }
+  }
+}

src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp

@@ -1,8 +1,8 @@
 #include "execution_gateway_provider.h"
-#include "binance_execution/cross_margin/cross_margin_gateway.h"
-#include "binance_execution/gateway_config.h"
-#include "binance_execution/spots/spots_gateway.h"
-#include "binance_execution/usds_futures/usds_futures_gateway.h"
+#include "binance/cross_margin/cross_margin_gateway.h"
+#include "binance/gateway_config.h"
+#include "binance/spots/spots_gateway.h"
+#include "binance/usds_futures/usds_futures_gateway.h"
 #include "okx_execution/okx_execution_gateway.h"
 #include "onixs_fix/OnixFIXAdapter.h"
 #include "python_interop/plugin_manager.h"
@@ -47,9 +47,8 @@ static std::unique_ptr<T> make_gw(
     const config::JetFIX::Connection &conn, Args &&...args) {
    LOG(OrderMan, Info,
        R"([create_exe_gw.json]){{ "name": "{}", "venue": "{}", "type": "{}", "ts": {} }})",
-       conn.name(), bts::api::venue_name_pretty(conn.venue()),
-       std::string(bts::utils::cached_type_name<T>()),
-       bts::core::TimeStamp::now_ns());
+       conn.name(), api::to_pretty(conn.venue()),
+       std::string(utils::cached_type_name<T>()), core::TimeStamp::now_ns());
    return std::make_unique<T>(std::forward<Args>(args)...);
 }
 
@@ -122,19 +121,19 @@ struct ExecutionGatewayProvider::PImpl {
          return make_gw<OkxExecutionGateway>(conn, o);
       case api::VENUE_BINANCE: {
          auto cfg =
-             parse_config_json<binance_execution::GatewayConfig>(conn.name());
+             parse_config_json<binance::GatewayConfig>(conn.name());
          cfg.expand_env();
          cfg.deduce_key_type();
          if (cfg.use_cross_margin_api) {
-            return make_gw<binance_execution::cross_margin::CrossMarginGateway>(
+            return make_gw<binance::cross_margin::CrossMarginGateway>(
                 conn, o, std::move(cfg), conn);
          }
-         return make_gw<binance_execution::spots::SpotsGateway>(
+         return make_gw<binance::spots::SpotsGateway>(
              conn, o, std::move(cfg), conn);
       }
       case api::VENUE_BINANCE_USDS_FUTURES: {
          auto cfg =
-             parse_config_json<binance_execution::GatewayConfig>(conn.name());
+             parse_config_json<binance::GatewayConfig>(conn.name());
          cfg.expand_env();
          if (cfg.use_cross_margin_api) {
             WARN_FN(
@@ -142,7 +141,7 @@ struct ExecutionGatewayProvider::PImpl {
                 "Config has 'use_cross_margin_api' set to true, but this is "
                 "only supported for spots (using regular api).");
          }
-         return make_gw<binance_execution::usds_futures::UsdsFuturesGateway>(
+         return make_gw<binance::usds_futures::UsdsFuturesGateway>(
              conn, o, std::move(cfg), conn);
       }
       default:

src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp

@@ -369,7 +369,7 @@ struct PyExecutionGatewayAdapter::PImpl {
        bts::md::py_interop::PySecurity &py_sec, const bts::api::Security &sec) {
       py_sec.set_id(sec.id());
       py_sec.set_asset(sec.symbol());
-      py_sec.set_venue(bts::api::venue_name_pretty(sec.exchange()));
+      py_sec.set_venue(bts::api::to_pretty(sec.exchange()).name());
       py_sec.set_currency(sec.currency());
       // Exchange ID and name
       std::string name = bts::securities::descriptor(sec);
@@ -442,9 +442,8 @@ struct PyExecutionGatewayAdapter::PImpl {
    void to_py(PyCreateOrderRequest &py_req, const CreateOrderMsg &msg) {
       py_req.set_key(msg.key.value());
       py_req.set_exchange_security_id(msg.security_name());
-      py_req.set_venue(bts::api::venue_name_pretty(msg.venue()));
-      py_req.set_security_type(
-          bts::api::security_type_name_pretty(msg.security_type()));
+      py_req.set_venue(api::to_pretty(msg.venue()).name());
+      py_req.set_security_type(api::to_pretty(msg.security_type()).name());
       py_req.set_is_buy(msg.side == bts::api::SIDE_BUY);
       py_req.set_price(msg.price.value());
       py_req.set_quantity(msg.quantity.full_value_float());
@@ -462,9 +461,8 @@ struct PyExecutionGatewayAdapter::PImpl {
       py_req.set_exchange_order_id(
           std::to_string(msg.exchange_order_id.value()));
       py_req.set_exchange_security_id(cf.security_name());
-      py_req.set_venue(bts::api::venue_name_pretty(cf.venue()));
-      py_req.set_security_type(
-          bts::api::security_type_name_pretty(cf.security_type()));
+      py_req.set_venue(api::to_pretty(cf.venue()).name());
+      py_req.set_security_type(api::to_pretty(cf.security_type()).name());
       py_req.set_is_buy(cf.side == bts::api::SIDE_BUY);
       py_req.set_price(cf.price.value());
       py_req.set_quantity(cf.quantity.full_value_float());
@@ -475,9 +473,8 @@ struct PyExecutionGatewayAdapter::PImpl {
       py_req.set_exchange_order_id(
           std::to_string(msg.exchange_order_id.value()));
       py_req.set_exchange_security_id(msg.security_name());
-      py_req.set_venue(bts::api::venue_name_pretty(msg.venue()));
-      py_req.set_security_type(
-          bts::api::security_type_name_pretty(msg.security_type()));
+      py_req.set_venue(bts::api::to_pretty(msg.venue()).name());
+      py_req.set_security_type(api::to_pretty(msg.security_type()).name());
       py_req.set_is_buy(msg.side == bts::api::SIDE_BUY);
    }
 

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp

@@ -293,6 +293,24 @@ struct BinanceUsdsFuturesSubscription::PImpl {
       return it->second;
    }
 
+   /**
+   * @brief wrapper method to batch securities so sub limits are not reached
+   */
+   template <class SecDescs>
+   void batch_subscribe_market_data(SecDescs securities, size_t n = 16) {
+      auto it = securities.begin();
+      while (it != securities.end()) {
+         auto batch_end = std::next(
+             it,
+             std::min(
+                 n, static_cast<size_t>(std::distance(it, securities.end()))));
+         SecDescs batch(it, batch_end);
+         subscribe_market_data(batch);
+         it = batch_end;
+         std::this_thread::sleep_for(500ms);
+      }
+   }
+
    /**
     * @brief Subscribes to depth, ticker, and price data for a list of
     * securities
@@ -358,7 +376,11 @@ struct BinanceUsdsFuturesSubscription::PImpl {
       }
 
       listener->on_securities(sec_ptrs);
-      subscribe_market_data(sec_descs);
+      if (cfg.use_batch_subscriptions()) {
+         batch_subscribe_market_data(sec_descs);
+      } else {
+         subscribe_market_data(sec_descs);
+      }
    }
 };
 

src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.cpp

@@ -1,5 +1,6 @@
 #include "CCApiMarketDataProvider.h"
-#include "spark/api/domain/Venue.pb.h"
+#include "binance/usds_futures/usds_futures_md_source.h"
+#include "spark/api/domain/Venue.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/config/config.h"
 #include "spark/io/ProtoPrinter.h"
@@ -7,6 +8,8 @@
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/MDProducer.h"
+#include "spark/md/SymbolLoader.h"
+#include "spark/md/UserSymbol.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance/BinanceSubscription.h"
 #include "sparkd/md/binance_us/BinanceUsSubscription.h"
@@ -82,56 +85,88 @@ CCApiMarketDataProvider::get_sources() {
           "CCApiMarketDataProvider::get_sources failed. Invalid config: {}",
           fmt::to_string(fmt::join(cfg_errs, "\n")));
    }
-
+   std::set<UserSymbol> syms;
+   auto is_venue_enabled = [&syms](api::domain::Venue venue) {
+      syms.clear();
+      SymbolLoader::load(syms, domain::SecurityCriteria::create({venue}));
+      if (syms.empty()) {
+         LOG(md, Warn,
+             R"([ccapi_md_pro_disabled.json]{{ "venue": "{}", "rsn": "No symbols config", "ts": {} }})",
+             venue, core::TimeStamp::now_ns());
+         return false;
+      }
+      return true;
+   };
    // Binance
-   for (const auto &binance : cfg.binance()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_BINANCE,
-          std::make_shared<binance::BinanceSubscription>(
-              &me_->dispatcher, binance, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_BINANCE)) {
+      for (const auto &binance : cfg.binance()) {
+         me_->sources.emplace_back(
+             api::domain::VENUE_BINANCE,
+             std::make_shared<binance::BinanceSubscription>(
+                 &me_->dispatcher, binance, me_->sec_id_store.get()));
+      }
    }
    // Binance US
-   for (const auto &binance_us : cfg.binance_us()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_BINANCE_US,
-          std::make_shared<binance_us::BinanceUsSubscription>(
-              &me_->dispatcher, binance_us, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_BINANCE_US)) {
+      for (const auto &binance_us : cfg.binance_us()) {
+         me_->sources.emplace_back(
+             api::domain::VENUE_BINANCE_US,
+             std::make_shared<binance_us::BinanceUsSubscription>(
+                 &me_->dispatcher, binance_us, me_->sec_id_store.get()));
+      }
    }
    // Binance USDS Futures
-   for (const auto &binance_ufs : cfg.binance_usds_futures()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_BINANCE_USDS_FUTURES,
-          std::make_shared<
-              binance_usds_futures::BinanceUsdsFuturesSubscription>(
-              &me_->dispatcher, binance_ufs, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_BINANCE_USDS_FUTURES)) {
+      for (const auto &binance_ufs : cfg.binance_usds_futures()) {
+         if (binance_ufs.use_ccapi()) {
+            me_->sources.emplace_back(
+                api::domain::VENUE_BINANCE_USDS_FUTURES,
+                std::make_shared<
+                    binance_usds_futures::BinanceUsdsFuturesSubscription>(
+                    &me_->dispatcher, binance_ufs, me_->sec_id_store.get()));
+         } else {
+            me_->sources.emplace_back(
+                api::domain::VENUE_BINANCE_USDS_FUTURES,
+                std::make_shared<::binance::usds_futures::UsdsFuturesMdSource>(
+                    binance_ufs, me_->sec_id_store));
+         }
+      }
    }
    // Coinbase
-   for (const auto &coinbase : cfg.coinbase()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_COINBASE,
-          std::make_shared<coinbase::CoinbaseSubscription>(
-              &me_->dispatcher, coinbase, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_COINBASE)) {
+      for (const auto &coinbase : cfg.coinbase()) {
+         me_->sources.emplace_back(
+             api::domain::VENUE_COINBASE,
+             std::make_shared<coinbase::CoinbaseSubscription>(
+                 &me_->dispatcher, coinbase, me_->sec_id_store.get()));
+      }
    }
    // Deribit
-   for (const auto &deribit : cfg.deribit()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_DERIBIT,
-          std::make_shared<deribit::DeribitSubscription>(
-              &me_->dispatcher, deribit, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_DERIBIT)) {
+      for (const auto &deribit : cfg.deribit()) {
+         me_->sources.emplace_back(
+             api::domain::VENUE_DERIBIT,
+             std::make_shared<deribit::DeribitSubscription>(
+                 &me_->dispatcher, deribit, me_->sec_id_store.get()));
+      }
    }
    // FTX
-   for (const auto &ftx : cfg.ftx()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_FTX,
-          std::make_shared<ftx::FTXSubscription>(
-              &me_->dispatcher, ftx, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_FTX)) {
+      for (const auto &ftx : cfg.ftx()) {
+         me_->sources.emplace_back(
+             api::domain::VENUE_FTX,
+             std::make_shared<ftx::FTXSubscription>(
+                 &me_->dispatcher, ftx, me_->sec_id_store.get()));
+      }
    }
    // Okx
-   for (const auto &okx : cfg.okx()) {
-      me_->sources.emplace_back(
-          api::domain::VENUE_OKX,
-          std::make_shared<okx::OkxSubscription>(
-              &me_->dispatcher, okx, me_->sec_id_store.get()));
+   if (is_venue_enabled(api::domain::VENUE_OKX)) {
+      for (const auto &okx : cfg.okx()) {
+         me_->sources.emplace_back(
+             api::domain::VENUE_OKX,
+             std::make_shared<okx::OkxSubscription>(
+                 &me_->dispatcher, okx, me_->sec_id_store.get()));
+      }
    }
    return me_->sources;
 }

src/app_sparkd/sparkd/md/recording/MDReplayer.cpp

@@ -1,6 +1,7 @@
 #include "MDReplayer.h"
 #include "spark/core/TimeStamp.h"
 #include "spark/domain/Book.h"
+#include "spark/domain/DomainSecurityUtils.h"
 #include "spark/domain/Price.h"
 #include "spark/domain/Quantity.h"
 #include "spark/domain/Security.h"

src/client/spark/Daemons/MDDaemon.cs

@@ -141,8 +141,8 @@ namespace bts {
 
          (TickDisplaySettingsImmutable? Price, TickDisplaySettingsImmutable? Quantity) GetDisplaySettings(SecurityImmutable sec) {
             var meta = _security_metadata.get(sec);
-            var priceDisplay = meta?.PriceDisplay ?? Security.Defaults.price_display_settingsi;
-            var qtyDisplay = meta?.QuantityDisplay ?? Security.Defaults.quantity_display_settingsi;
+            var priceDisplay = meta.HasValue && meta.Value.HasPriceDisplay ? meta.Value.PriceDisplay : Security.Defaults.price_display_settingsi;
+            var qtyDisplay = meta.HasValue && meta.Value.HasQuantityDisplay ? meta.Value.QuantityDisplay : Security.Defaults.quantity_display_settingsi;
             if (sec.HasContractSpec) {
                var pd = priceDisplay.CloneToMutable();
                pd.TickFactors = TickUtils.get_tick(sec, 0).to_api_tick_factors();

src/client/spark/Model/Settings/SettingsPersistenceModel.cs

@@ -103,9 +103,9 @@ namespace mbts.persist {
          var re = new Regex(pattern);
          return _persist
             .subscribe_ns(_user_ns(ns))
+            .Filter(r => re.IsMatch(r.Key))
             .Select(update =>
-               update.added
-                  .Where(item => re.IsMatch(item.Key))
+               update.all
                   .Select(item => new PersistResult<T>() {
                      user = _user,
                      ns = ns,

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -199,8 +199,8 @@ namespace mbts.spark {
          add_col(ColKey.Theta, r => fmt_greek(r.theta), nameof(Row.theta), theta_filter, FilterType.DoubleMinMax, ColWidth.Greeks);
          add_col(ColKey.Rho, r => fmt_greek(r.rho), nameof(Row.rho), rho_filter, FilterType.DoubleMinMax, ColWidth.Greeks);
          add_col(ColKey.CashPerPoint, r => r.cash_per_point.ToString("0.00#####"), nameof(Row.cash_per_point), cash_per_point_filter, FilterType.DoubleMinMax, ColWidth.Greeks);
-         add_col(ColKey.MinOrderQuantity, r => r.min_order_quantity.ToString("0.00#####"), nameof(Row.min_order_quantity), min_order_quantity_filter, FilterType.DoubleMinMax, ColWidth.Qty);
-         add_col(ColKey.MinOrderNotional, r => r.min_order_notional.ToString("0.00#####"), nameof(Row.min_order_notional), min_order_notional_filter, FilterType.DoubleMinMax, ColWidth.Qty);
+         add_col(ColKey.MinOrderQuantity, r => r.security.format_quantity(r.min_order_quantity), nameof(Row.min_order_quantity), min_order_quantity_filter, FilterType.DoubleMinMax, ColWidth.Qty);
+         add_col(ColKey.MinOrderNotional, r => r.security.format_price(r.min_order_notional), nameof(Row.min_order_notional), min_order_notional_filter, FilterType.DoubleMinMax, ColWidth.Qty);
 
          // row vm filter
          // Althoguh securities are filtered at the row 'model' level (SecurityMD), the column base filters have to be applied at the row vm level,
@@ -448,6 +448,7 @@ namespace mbts.spark {
 
       public static IComparer<Row> get_column_comparer(ColKey column) {
          return column switch {
+            ColKey.Venue => wrap(r => r.venue.ToString()),
             ColKey.Security => wrap(r => r.security),
             ColKey.SecurityKey => wrap(r => r.security.Id),
             ColKey.Volume => wrap(r => r.volume),

src/client/spark/Modules/Trading/Fills/FillsColumn.cs

@@ -36,6 +36,7 @@ namespace mbts.spark {
    class FillsColumn<TColFilter> : DIYGrid.RenderTextColumn<FillsRowVM, ColKey>, IFillsColumn, IFilterableDIYGridColumn<FillTuple, ColKey, FillsFilterStates> {
       readonly Func<FillsFilterStates, TColFilter> _get_col_filter;
       readonly Action<FillsFilterStates, TColFilter> _set_col_filter;
+      readonly Func<FillsRowVM, string> getText;
 
       public FillsColumn(
          ColKey key,
@@ -57,16 +58,20 @@ namespace mbts.spark {
          _get_col_filter = get_col_filter;
          _set_col_filter = set_col_filter;
          this.filter_model = filter_model;
+         getText = get_text;
       }
 
       public new IDataGridColumnPersistentFilter<FillTuple, TColFilter> filter_model { get; }
 
-      IDataGridColumnFilter<FillTuple> IFilterableDIYGridColumn<FillTuple, ColKey>.filter_model => filter_model;
+      IDataGridColumnFilter <FillTuple> IFilterableDIYGridColumn<FillTuple, ColKey>.filter_model => filter_model;
       IDataGridColumnFilter IFilterableDIYGridColumn.filter_model => filter_model;
       ColKey IDIYGridColumn<ColKey>.id => key;
 
       DIYGrid.ColumnBase IFillsColumn.column_view => this;
 
+      public string GetText(FillsRowVM row) =>
+         getText(row);
+
       public FillsFilterStates assign_filter_settings(FillsFilterStates filter_settings) {
          _set_col_filter(filter_settings, filter_model.get_state());
          return filter_settings;

src/client/spark/Modules/Trading/Fills/FillsOrderTypeColumn.cs

@@ -17,9 +17,11 @@ namespace mbts.spark {
    class FillsOrderTypeColumn<TFilter> : DIYGrid.OrderTypeColumn<ColKey>, IFillsColumn {
       readonly Func<FillsFilterStates, TFilter> _get_col_filter;
       readonly Action<FillsFilterStates, TFilter> _set_col_filter;
+      readonly Func<FillsRowVM, string> getText;
 
       public FillsOrderTypeColumn(
          ColKey key,
+         Func<FillsRowVM, string> getText,
          BindingBase badge_fill_binding,
          Binding display_name_binding,
          IDataGridColumnPersistentFilter<FillTuple, TFilter> filter_model,
@@ -27,6 +29,7 @@ namespace mbts.spark {
          Action<FillsFilterStates, TFilter> set_col_filter,
          string header = "Order Type")
          : base(key, badge_fill_binding, display_name_binding, filter_model, header) {
+         this.getText = getText;
          this.filter_model = filter_model;
          _get_col_filter = get_col_filter;
          _set_col_filter = set_col_filter;
@@ -40,6 +43,9 @@ namespace mbts.spark {
       IDataGridColumnFilter IFilterableDIYGridColumn.filter_model => filter_model;
       ColKey IDIYGridColumn<ColKey>.id => key;
 
+      public string GetText(FillsRowVM r) =>
+         getText(r);
+
       public FillsFilterStates assign_filter_settings(FillsFilterStates filter_settings) {
          _set_col_filter(filter_settings, filter_model.get_state());
          return filter_settings;

src/client/spark/Modules/Trading/Fills/FillsSideColumn.cs

@@ -24,6 +24,9 @@ namespace mbts.spark {
 
       FillsView.Columns IDIYGridColumn<FillsView.Columns>.id => key;
 
+      public string GetText(FillsRowVM row) =>
+         row.side.ToString();
+
       public FillsFilterStates assign_filter_settings(FillsFilterStates filter_settings) {
          filter_settings.side = filter_model.get_state();
          return filter_settings;

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -30,6 +30,7 @@ namespace mbts.spark {
    using System.Collections.Specialized;
    using System.Linq;
    using System.Reactive.Linq;
+   using System.Text;
    using System.Windows.Data;
    using System.Windows.Media;
 
@@ -138,6 +139,7 @@ namespace mbts.spark {
          _columns.Add(
             new FillsOrderTypeColumn<OrderType[]>(
                Columns.OrderType,
+               r => r.order_type.ToString(),
                new Binding(nameof(FillsRowVM.badge_fill)),
                new Binding(nameof(FillsRowVM.display_name)),
                order_type_filter,
@@ -161,6 +163,7 @@ namespace mbts.spark {
          _columns.Add(
             new FillsOrderTypeColumn<AlgoID[]>(
                Columns.RootAlgoType,
+               r => r.root_algo_order_type.ToString(),
                new Binding(nameof(FillsRowVM.root_algo_badge_color)) {
                   Converter = new ui.converters.ColorToBrushConverter()
                },
@@ -289,6 +292,16 @@ namespace mbts.spark {
             (fs, cf) => fs.fill_key = cf,
             DefaultColumnWidths.FillKey);
 
+         add_col(
+            Columns.IsAggressor,
+            DefaultColumnHeaders.IsAggressor,
+            r => r.is_aggressor ? SpecialCharacters.CheckMark : string.Empty,
+            dv_filter(new[] { false, true }, r => r.is_aggressor),
+            DataGridFilterKnownTemplates.YesNo,
+            fs => fs.is_aggressor,
+            (fs, cf) => fs.is_aggressor = cf,
+            DefaultColumnWidths.IsAggressor);
+
          add_col(
             Columns.OrderKey,
             DefaultColumnHeaders.OrderKey,
@@ -464,6 +477,33 @@ namespace mbts.spark {
       public void on_sorting(object sender, DIYGridSortingEventArgs e) =>
          _rows_model.on_sorting(e);
 
+      public IEnumerable<string> CsvRows(bool includeHeaderRow) {
+         var cols = _columns.AsCollection().ToArray();
+         if (cols.Length == 0) {
+            yield break;
+         }
+         var sb = new StringBuilder();
+         if (includeHeaderRow) {
+            for (int i = 0; i < cols.Length; i++) {
+               if (i > 0) {
+                  sb.Append(',');
+               }
+               sb.Append(cols[i].id.ToString());
+            }
+            yield return sb.ToString();
+         }
+         foreach (var r in rows) {
+            sb.Clear();
+            for (int i = 0; i < cols.Length; i++) {
+               if (i > 0) {
+                  sb.Append(',');
+               }
+               sb.Append(cols[i].GetText(r));
+            }
+            yield return sb.ToString();
+         }
+      }
+
       static IObservable<TableUpdate<FillTuple, string>> _get_observable(
          OrdersModel model, FillsSettings state) {
          var orders_hash = state.orders?.ToHashSet();
@@ -483,6 +523,7 @@ namespace mbts.spark {
             case Columns.FillKind: return wrap_default(ft => ft.fill.FillKind);
             case Columns.FillStatus: return wrap_default(ft => ft.fill.FillStatus);
             case Columns.FillTime: return wrap(ft => ft.fill.Timestamp.to_local_time());
+            case Columns.IsAggressor: return wrap_default(ft => ft.fill.FillAggressorFlag == FillAggressorFlag.Aggressor);
             case Columns.OrderKey: return wrap(ft => ft.fill.OrderKey);
             case Columns.OrderType: return wrap_default(ft => ft.order?.order_type() ?? OrderType.None);
             case Columns.RootAlgoType: return wrap_default(ft => ft.root_algo.HasValue ? ft.root_algo.Value.value.CustomCase : OrderType.None);

src/client/spark/Modules/Trading/Fills/FillsView.xaml.cs

@@ -22,6 +22,7 @@ namespace mbts.ui.spark {
          Price,
          Size,
          NotionalValue,
+         IsAggressor,
          FillKind,
          FillStatus,
          FillKey,

src/client/spark/Modules/Trading/Fills/FillsWindow.cs

@@ -17,6 +17,7 @@ using Spark.Core.Disposables;
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Windows;
@@ -91,6 +92,7 @@ namespace mbts.spark {
                has_selected_rows_src,
                _ => _,
                _ => _vm.selected_rows.Select(r => r.security_key).ToImmutableHashSet()),
+            new SparkCommandCommand("Export to CSV...", Icons.Disk, FileDialogCommand.ExportToCsv(ExportToCsv)),
             // wrap the grid settings view factory in a dialogCommand here since the vm is destroyed/recreated when the security criteria changes
             SparkCommands.window_settings_command(new DialogCommand(() => {
                var vm = new ShellSettingsDialogVM<GridSettings, SimpleGridSettingsVM>(
@@ -128,6 +130,14 @@ namespace mbts.spark {
          //       show_ladder_cmd,
          //       SparkCommands.KeyGestures.ShowLadder);
          // }
+
+         void ExportToCsv(string filePath) {
+            using var fs = File.OpenWrite(filePath);
+            using var sw = new StreamWriter(fs);
+            foreach (var row in _vm.CsvRows(true)) {
+               sw.WriteLine(row);
+            }
+         }
       }
 
       protected override void OnParentViewCreated(SparkWindowView parent_view) =>

src/client/spark/Modules/Trading/Fills/IFillsColumn.cs

@@ -8,6 +8,7 @@ namespace mbts.spark {
 
    interface IFillsColumn : IFilterableDIYGridColumn<FillTuple, FillsView.Columns, FillsFilterStates>, ISortableDIYGridColumn<FillsView.Columns> {
       DIYGrid.ColumnBase column_view { get; }
+      string GetText(FillsRowVM row);
    }
 
    static class FillsColumnExtensions {

src/client/spark/Modules/Trading/Fills/Settings/FillsFilterStates.cs

@@ -90,6 +90,10 @@ namespace bts.shell.fills {
       [DIYGridColumnID(nameof(Columns.FillStatus))]
       public Fill.Types.FillStatus[] fill_status { get; set; }
 
+      [ProtoMember(19)]
+      [DIYGridColumnID(nameof(Columns.IsAggressor))]
+      public bool[] is_aggressor { get; set; }
+
       // PNL's Expiration group popups use this prop to filtered data by a particular exp (it isn't
       // serialized bc it's not part of the main window's settings)
       public ExpirationDate? exp_date { get; set; }
@@ -116,6 +120,7 @@ namespace bts.shell.fills {
          pb.add_discrete_values(fs.fill_kind, ft => ft.fill.FillKind);
          pb.add_discrete_values(fs.symbol, ft => ft.security?.Symbol ?? string.Empty);
          pb.add_discrete_values(fs.fill_status, ft => ft.fill.FillStatus);
+         pb.add_discrete_values(fs.is_aggressor, ft => ft.fill.FillAggressorFlag == FillAggressorFlag.Aggressor);
          // numeric
          pb.add_numeric(fs.size, ft => ft.fill.LastShares);
          pb.add_numeric(fs.notional_value, ft => ft.fill.notional_value());

src/client/spark/Modules/Trading/Fills/Settings/FillsSettings.cs

@@ -59,6 +59,7 @@ namespace bts.shell.fills {
          public static GridColumnSettings side => new GridColumnSettings(col_id(Columns.Side), DefaultColumnHeaders.Side, width: DefaultColumnWidths.Side);
          public static GridColumnSettings quantity => new GridColumnSettings(col_id(Columns.Size), DefaultColumnHeaders.Qty, width: DefaultColumnWidths.Qty, horizontal_alignment: System.Windows.HorizontalAlignment.Right);
          public static GridColumnSettings price => new GridColumnSettings(col_id(Columns.Price), DefaultColumnHeaders.Price, width: DefaultColumnWidths.Price, horizontal_alignment: System.Windows.HorizontalAlignment.Right);
+         public static GridColumnSettings is_aggressor => new GridColumnSettings(col_id(Columns.IsAggressor), DefaultColumnHeaders.IsAggressor, width: DefaultColumnWidths.CheckMark);
          public static GridColumnSettings fill_kind => new GridColumnSettings(col_id(Columns.FillKind), DefaultColumnHeaders.FillKind, width: DefaultColumnWidths.FillKind);
          public static GridColumnSettings fill_status => new GridColumnSettings(col_id(Columns.FillStatus), DefaultColumnHeaders.FillStatus, width: DefaultColumnWidths.FillStatus);
          public static GridColumnSettings fill_key => new GridColumnSettings(col_id(Columns.FillKey), DefaultColumnHeaders.FillKey, width: DefaultColumnWidths.FillKey);
@@ -84,6 +85,7 @@ namespace bts.shell.fills {
                yield return side;
                yield return quantity;
                yield return price;
+               yield return is_aggressor;
                yield return fill_kind;
                yield return fill_key;
                yield return order_key;

src/client/spark/Modules/Trading/PNL/VM/PNLCellEmbeddedAnimator.cs

@@ -28,7 +28,9 @@ namespace bts.shell.pnl {
          foreach (var t in _triggers) {
             foreach (var kv in t.trigger(data)) {
                if (!bd.TryGetValue(kv.Key, out var app)) {
-                  app = _defaults[kv.Key];
+                  if (!_defaults.TryGetValue(kv.Key, out app)) {
+                     continue;
+                  }
                }
                app = kv.Value.merge(app);
                bd[kv.Key] = app;

src/client/spark/Risk/PNL/PNLTradeLogRow.cs

@@ -169,6 +169,8 @@ namespace mbts.analytics {
             return 0;
          }
       }
+      // if < 0 then rebate, > 0 we pay a fee
+      public double trade_transaction_fee => 0;
       public double pos_gamma {
          get {
             return tl_row.security_type == SecurityType.Option ?

src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs

@@ -51,6 +51,7 @@ namespace mbts.ui.controls {
             TimeOfDay = 84,
             TimeInForce = 47,
             User = 53,
+            IsAggressor = 64,
             Aggressor = 80,
             Volume = 74,
             FillKind = 70,
@@ -149,6 +150,7 @@ namespace mbts.ui.controls {
             FilledQuantity = "Fill Qty",
             FilledNotionalValue = "Fill Notional",
             FillStatus = "Fill Status",
+            IsAggressor = "Aggressor",
             Name = "Name",
             Description = "Description",
             N_Legs = "# Legs",

src/client/spark/view/Extensions/Command/FileDialogCommand.cs

@@ -15,6 +15,19 @@ namespace mbts.ui {
       public static IDialogCommand open_file(Action<string> execute, Func<bool> can_execute, string initial_dir, params (string name, string[] extensions)[] extensions) =>
          new AnonymousFileDialogCommand<object>(_ => _create_open_file_dialog(null, extensions), (_, file) => execute(file), _ => can_execute());
 
+      public static IDialogCommand ExportToCsv(Action<string> execute, Func<bool> canExecute = null, string initialDir = null, string initialName = null) {
+         return new AnonymousFileDialogCommand<object>(DialogFactory, Execute, CanExecute);
+
+         SaveFileDialog DialogFactory(object _) =>
+            _create_save_file_dialog(initialDir, initialName, new[] { ("Excel", new[] { "csv" }) });
+
+         void Execute(object _, string file) =>
+            execute(file);
+
+         bool CanExecute(object _) =>
+            canExecute?.Invoke() ?? true;
+      }
+
       public static IDialogCommand export_proto<TProto>(
          Func<TProto> get_proto,
          Func<bool> can_execute = null,
@@ -66,7 +79,7 @@ namespace mbts.ui {
             }
          }, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));
 
-      static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
+      public static TDialog CreateFileDialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
          var rv = new TDialog();
          if (!string.IsNullOrEmpty(initial_dir))
             rv.InitialDirectory = initial_dir;
@@ -78,7 +91,7 @@ namespace mbts.ui {
       }
 
       static SaveFileDialog _create_save_file_dialog(string initial_dir = null, string initial_name = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) {
-         var rv = _create_file_dialog<SaveFileDialog>(initial_dir, filter_items);
+         var rv = CreateFileDialog<SaveFileDialog>(initial_dir, filter_items);
          rv.OverwritePrompt = true;
          rv.AddExtension = true;
          if (!string.IsNullOrEmpty(initial_name))
@@ -87,7 +100,7 @@ namespace mbts.ui {
       }
 
       static OpenFileDialog _create_open_file_dialog(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) =>
-         _create_file_dialog<OpenFileDialog>(initial_dir, filter_items);
+         CreateFileDialog<OpenFileDialog>(initial_dir, filter_items);
 
       static string _build_filter(IEnumerable<(string name, string[] extensions)> items) =>
          string.Join("|", items.Select(i => {