refactoring onix options

This reverts commit 4f05b95bd0.

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "externals"]
 	path = externals
 	url = git@bitbucket.org:bluetradesys/externals.git
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git

CMakePresets.json

@@ -26,6 +26,7 @@
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
         "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
+        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
         "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
         "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
         "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",

notes

@@ -0,0 +1,6 @@
+cd ~/spark/build/clang/debug/; cmake --build . --target install; rm -rf ~/spark/install/Debug/bin.zip; tar -zcvf  ~/spark/install/Debug/bin.zip ~/spark/install/Debug/bin; scp ~/spark/install/Debug/bin.zip bts@10.1.10.31:~/13000/; cd ~/spark/
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; docker container stop spark_13000; docker container start spark_13000
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; cp -r sparkbin ~/.sparkd-dev/mounts/sparkd_build/; 
+ES SECURITYTYPE_FUTURE at VENUE_GLBX
+./sparkd md dump --symbol ES --type future --venue glbx --platform mdp3onixsex
+BT5u53r

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionTokens.lex

@@ -14,7 +14,7 @@ ParenthesisClose   \)
 Add                \+
 Subtract           \-
 Multiply           \*
-Divide             \\
+Divide             /
 Equals             =
 Comma              ,
 

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -2,6 +2,8 @@ namespace Bts.BooleanExpressions.Arguments;
 
 using bts;
 
+using Spark.Core.Collections;
+
 static class ArgumentComparer {
    public static int Compare(
       IArgument value1, IArgument value2, IDictionary<string,object?> env) {
@@ -14,7 +16,10 @@ static class ArgumentComparer {
       var cmp1 = value1.Cast<IComparable>();
       var cmp2 = value2.Cast<IComparable>();
       try {
-         return cmp1.GetValue(env).CompareTo(cmp2.GetValue(env));
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;

src/DataStreamSuite/BooleanExpressions/BooleanExpressionGrammar.y

@@ -44,7 +44,7 @@
 %%
 
 BoundExpression : Definitions Expression { $$ = new($2, $1); }
-   | Expression;
+   | Expression { $$ = new($1, ImmutableList<Definition>.Empty); };
 
 Definitions :
    Definition { $$ = ImmutableList<Definition>.Empty.Add($1); }
@@ -57,46 +57,70 @@ Definition :
 
 Expression :
    Expression OR Expression {
-      $$ = new(env => $1.Evaluate(env) || $3.Evaluate(env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) || right.Evaluate(env));
    }
    | Expression AND Expression {
-      $$ = new(env => $1.Evaluate(env) && $3.Evaluate(env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) && right.Evaluate(env));
    }
    | Expression XOR Expression {
-      $$ = new(env => $1.Evaluate(env) != $3.Evaluate(env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) != right.Evaluate(env));
    }
    | NOT Expression {
-      $$ = new(env => !$2.Evaluate(env));
+      var exp = $2;
+      $$ = new(env => !exp.Evaluate(env));
    }
    | PARENTHESIS_OPEN Expression PARENTHESIS_CLOSE {
       $$ = $2; 
    }
    | Argument GREATER Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) > 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) > 0);
    }
    | Argument GREATER_EQUAL Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) >= 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) >= 0);
    }
    | Argument EQUAL Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) == 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) == 0);
    }
    | Argument LESS_EQUAL Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) <= 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) <= 0);
    }
    | Argument LESS Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) < 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) < 0);
    }
    | Argument DIFFERENT Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) != 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
    }
    | Argument DIFFERENT2 Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) != 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
    }
    | Argument MATCHES Argument {
-      $$ = new(env => Arguments.ArgumentMatcher.Match($1, $3, env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentMatcher.Match(left, right, env));
    }
    | Argument {
-      $$ = new(env => (bool)$1.GetValue(env));
+      var arg = $1;
+      $$ = new(env => (bool)arg.GetValue(env));
    }
    ;
 

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -7,7 +7,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -0,0 +1,32 @@
+using BCG;
+
+namespace BCG
+{
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("add_to_cloud")]
+   public class AddToCloudStream : ViewStream
+   {
+      private readonly string _name, _stream_exp;
+      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
+      public AddToCloudStream(string name, params string[] lines)
+      : this(name, 0, lines)
+      {
+      }
+
+      public AddToCloudStream(string name, int persist, params string[] lines)
+      {
+         _name = name;
+         _stream_exp = getExpression(lines);
+         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
+         mySetRealSource(new TableStream("cloud_streams"));
+      }
+
+      static string getExpression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -0,0 +1,100 @@
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
+using Bts.DataStreamExpressions;
+using System.Collections.Generic;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+   // listens to the requests posted on "cloud_streams" request table, and 
+   // if unfulfilled starts a ViewPubsliher
+   public class CloudStreamManager : IWorker
+   {
+      public class StreamPubUnit
+      {
+         public string name;
+         public string stream_expression;
+         public ViewStream stream;
+         public bool persist;
+         public ViewPublisher view_publisher;
+
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
+         {
+            name = name_;
+            stream_expression = stream_expression_;
+            stream = stream_;
+            persist = persist_;
+            view_publisher = view_publisher_;
+         }
+      }
+      static ViewStream build_stream(string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         }
+         catch (Exception ex)
+         {
+            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+      private IDictionary<string, StreamPubUnit>
+         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+
+      public CloudStreamManager()
+      {
+         _stream.onChange += _on_str;
+
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         foreach (IBlueRow r in dp.newRows)
+         {
+            string name = r.get<string>("name");
+            bool persist = r.get<bool>("persist");
+            string stream_exp = r.get<string>("stream_exp");
+            if (!_stream_pub_map.ContainsKey(name))
+            {
+               _assign(name, stream_exp, persist);
+            }
+         }
+      }
+
+      private void _assign(string name, string stream_expression, bool persist)
+      {
+         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
+         ViewStream stream = build_stream(stream_expression);
+         _expose_to_ws(name, stream);
+         if (persist)
+         {
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
+         }
+         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
+         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
+         _stream_pub_map.Add(name, u);
+      }
+
+      private void _expose_to_ws(string name, ViewStream stream)
+      {
+         // TODO: john
+      }
+
+      public void doWork()
+      {
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -0,0 +1,59 @@
+using BCG;
+
+using System.Linq;
+
+namespace BCG
+{
+   public class StreamOffloadRequestor
+   {
+      public class OffloadOp
+      {
+         string name;
+         string stream_exp;
+      }
+      private IBlueTableSchema _schema;
+      private IBlueRowCollection _rows;
+      private PubTableBroker _broker = new PubTableBroker();
+      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+      public StreamOffloadRequestor()
+      {
+         _schema = new BlueTableSchema("cloud_streams",
+                                   new ColumnInfo("id", 0, typeof(int)),
+                                   new ColumnInfo("name", 1, typeof(string)),
+                                   new ColumnInfo("persist", 2, typeof(bool)),
+                                   new ColumnInfo("stream_exp", 3, typeof(string)));
+         _existing_offloads_vs.onChange += _on_str;
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         _rows = dp.newRows;
+         // this is needed so the broker knows the state of the table
+         _broker.createTable(_rows);
+      }
+      public void add_to_cloud(string name, string stream_exp, bool persist)
+      {
+         // check if it exists
+         if (_rows == null)
+         {
+            BlueTable table = new BlueTable(_schema);
+            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+            table.addRow(nr);
+            _broker.createTable(table);
+            // create table  
+         }
+         else
+         {
+            if (exists(name))
+               return;
+            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+            _broker.addRow(nr);
+         }
+      }
+
+      bool exists(string name)
+      {
+         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,13 +1,13 @@
-// using System;
-// using System.Data;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Utilities.General;
-// namespace BCG
-// {
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG
+{
 
-//    public class AddColumnsStream : MapStream
-//    {
-//       public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef))  {}    
-//    }
-// }
+   public class AddColumnsStream : MapStream
+   {
+      public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef))  {}    
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/CaptureSnapshotStream.cs

@@ -1,40 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class CaptureSnapshotStream : ViewStream
-//    {
-//       public CaptureSnapshotStream(string basePath,ViewStream sourceStream)
-//       {
-//          myBasePath = basePath;
-//          mySource = sourceStream;
-//          Directory.CreateDirectory(myBasePath);
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DateTime now = DateTime.Now;
-//          new ViewStreamWaiter(new SaverStream(new BlueRowCollectionStaticStream(((DataPacket)newValue).newRows),
-//              Path.Combine(myBasePath, String.Format("{0}_{1}.ds", now.ToString("yyyyMMdd_HHmmss"),
-//              now.Millisecond)))).waitUntilArrived();
-//          myFireEvent(newValue);
-//       }
-
-//       private string myBasePath;
-//       private ViewStream mySource;
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using Google.Protobuf;
+using System.Globalization;
+
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
+   // File naming:
+   // token_epoch_yyyyMMdd_HHmmss
+   // File location: directory(passed)\token\
+   // directory passed on, it creates a folder with the token passed
+   // data:
+   // epochtime, size, serialized data
+   [DataStreamUserRespresentation("pers")]
+   public class PersistorStream : ViewStream {
+      private ViewStream _source_stream;
+      private string _token;
+      private FileStream _out_fs;
+      private IBlueDataSerializer _serializer;
+      private string _file_name = String.Empty;
+      private long _rollover_size;
+      private readonly string _directory_path;
+      private int _n_updates_in_current_file = 0;
+
+      public PersistorStream(ViewStream src, string token, string directory, int rollover_size) {
+         _source_stream = src;
+         _token = token;
+         _directory_path = directory + $@"\{token}\";
+         _open_new_file();
+         _rollover_size = rollover_size;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _source_stream.onChange += _on_change;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _source_stream.onChange -= _on_change;
+         _out_fs.Close();
+      }
+
+
+      public static DateTime parse_datetime_from_filename(string s) {
+         // allbooks_1697054438_2023-10-11_16-00-38
+         string[] toks = s.Split("_");
+         bts.Log.assert(toks.Length == 4);
+         string dt = $"{toks[2]}_{toks[3]}";
+         return parse_datetime(dt);
+      }
+
+      // this is used to parse the string passed the datetime texts passed to cons of the stream 
+      public static DateTime parse_datetime(string s) {
+         // for whatever reason "yyyy-MM-dd_HH-mm-ss" is not parsable using DateTime.ParseExact (it only gets
+         // the Date not the time), "yyyy-MM-dd_HH:mm:ss" is parsable
+         // however a file name can not have : in its name, therefore we need to parse "yyyy-MM-dd_HH-mm-ss"
+         DateTime rv = DateTime.ParseExact(s, PersistorStream.DateTime_format, CultureInfo.InvariantCulture);
+         string[] toks = s.Split("_");
+         string time_str = toks[1];
+         string[] time_toks = time_str.Split("-");
+         bts.Log.assert(time_toks.Length == 3);
+         int hh = Int32.Parse(time_toks[0]);
+         int mm = Int32.Parse(time_toks[1]);
+         int ss = Int32.Parse(time_toks[2]);
+         rv = rv.Date;
+         rv = rv.AddHours(hh);
+         rv = rv.AddMinutes(mm);
+         rv = rv.AddSeconds(ss);
+         return rv;
+      }
+
+      public static string DateTime_format {
+         get {
+            return "yyyy-MM-dd_HH-mm-ss";
+         }
+      }
+
+      public static long dt_to_epoch(DateTime dt) {
+         DateTimeOffset dtos = new DateTimeOffset(dt, TimeSpan.Zero);
+         return dtos.ToUnixTimeSeconds();
+      }
+      public static long epoch_now() {
+         return DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+      }
+
+      void _on_change(object new_value) {
+         DataPacket dp = new_value as DataPacket;
+         byte[] serialized_update = null;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         if (_write_create) {
+            // new files always starts with create
+            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+         } else
+            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
+
+         BinaryWriter bw = new BinaryWriter(_out_fs);
+         long utc_epoch = epoch_now();
+
+         //time, length of data, data
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
+         bw.Write(serialized_update);
+         _out_fs.Flush();
+         myFireEvent(dp);
+         _n_updates_in_current_file++;
+         if (_out_fs.Length > _rollover_size) {
+            _close_current_and_open_new();
+         }
+      }
+
+      void _close_current_and_open_new() {
+         _out_fs.Close();
+         _open_new_file();
+      }
+
+      void _open_new_file() {
+         var epoch = epoch_now();
+         _n_updates_in_current_file = 0;
+         // Check if the directory was successfully created
+         if (!Directory.Exists(_directory_path)) {
+            Directory.CreateDirectory(_directory_path);
+         }
+         _file_name = $"{_directory_path}{_token}_{epoch}_{DateTime.Now.ToString(DateTime_format)}";
+         _out_fs = new FileStream(_file_name, FileMode.OpenOrCreate);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -5,55 +5,104 @@ using System.Text;
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
 using BCG.Utilities.General;
-using BCG.Utilities;
+using bts.collections;
+using System.Reflection.Metadata;
 
-namespace BCG
-{
-   public delegate void PlaybackStreamProgress(long readFar, long totalBytesRead);
-   public class PlaybackStream : ViewStream, IWorker
-   {
 
-      public PlaybackStream(string fileName) : this(fileName, 1.0) { }
+namespace BCG {
 
-      private string myFileName;
-      private FileStream myFileReader;
-      private double myPlaybackSpeed;
-      private DateTime myPlaybackStart;
-      private DateTime myNextPacketTime;
-      private IClientDeserializer myDeserializer;
-      private ITableAccumulator myAccumulator;
+   public class PlaybackStream : ViewStream, IWorker {
+      private readonly PlaybackType _type;
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      // these are the packets already downloaded from the current file
+      private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+      //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+      private IEnumerable<string> _file_names;
+      private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+      private DateTime _playback_t0, _playback_t1;
+      private bool _all_files_read = false;
+      SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
 
-      private BlueTable myLastTableVersion;
-
-      public PlaybackStream(string fileName, double playbackSpeed)
-      {
-         myFileName = fileName;
-         myPlaybackSpeed = playbackSpeed;
+      public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+         _type = PlaybackType.Streaming;
+         _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+         _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+         string token_directory_name = $"{directory_name}\\{token}";
+         bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+         List<string> files_in_range = new List<string>();
+         if (Directory.Exists(token_directory_name)) {
+            _file_names =
+         Directory.GetFiles(token_directory_name).Where(f => {
+            DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+            return file_time > _playback_t0 && file_time < _playback_t1;
+         });
+         }
       }
 
-      protected override void myOnFirstSubscriber()
-      {         myPlaybackStart = DateTime.Now;
-         myNextPacketTime = DateTime.MinValue;
-         myFileReader = new FileStream(myFileName, FileMode.Open, FileAccess.Read, FileShare.Read);
-         myReadNextPacketTime();
-         ViewStreamDatabase.instance.addWorker(this);
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+         _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _snapshot_data.packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
       }
 
-      protected override void myOnLastSubscriber()
-      {
-         if (myFileReader != null)
-            myFileReader.Close();
+
+      private void _load_packets_from_current_file() {
+         int curr_idx = _current_fs.Item1;
+         FileStream curr_fs = _current_fs.Item2;
+         _packets = new List<PlaybackPacket>();
+         _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+      }
+
+      private bool current_file_all_replayed {
+         get {
+            bool rv = true;
+            _packets.ForEach(p => rv = rv && p.replayed);
+            return rv;
+         }
+      }
+
+      protected override void myOnLastSubscriber() {
+         var fs = _current_fs.Item2;
+         if (fs != null) {
+            fs.Close();
+         }
          ViewStreamDatabase.instance.removeWorker(this);
       }
 
-      protected void myReadNextPacketTime()
-      {
-         if (myFileReader.Position == myFileReader.Length)
-         {
-            myNextPacketTime = DateTime.MaxValue;
-            myFileReader.Dispose();
-            if (onFileExhausted != null)
-               onFileExhausted();
+      // public event PlaybackStreamProgress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
          }
          else
          {
@@ -64,52 +113,76 @@ namespace BCG
          }
       }
 
-      public event PlaybackStreamProgress onProgress;
-      public event Action onFileExhausted;
-
-
       #region IWorker Members
 
-      public void doWork()
-      {
-         if (myNextPacketTime <= DateTime.Now)
-         {
-            if (myDeserializer == null)
-            {
-               myDeserializer = new ClientUniversalDeserializer();
-               myAccumulator = new TableAccumulator(myFileName, null);
-            }
-
-            BinaryReader reader = new BinaryReader(myFileReader);
-            int numBytes = reader.ReadInt32();
-            byte[] thisData = reader.ReadBytes(numBytes);
-            byte[] ops;
-            Array resultRowCollections;
-            myDeserializer.deserialize(new Stream[] { new MemoryStream(thisData) }, out ops, out resultRowCollections);
-
-            //there should only be one op in there becuae we just read a single update.
-            // BlueEventLog.logAssert(ops.Length == 1,"Playback stream got " +ops.Length.ToString() + " op codes in one update, from stream file "+myFileName);
-            //there shoudl only be one set of collections becaue there was just a single op code 
-            //(those two arrays are parallel)
-            // BlueEventLog.logAssert(resultRowCollections.Length == 1, "StreamOpCode was create, serializer should return exactly 1 set of rows");
-
-            if (ops[0] == (byte)StreamOpCode.create)
-            {
-               myLastTableVersion = new BlueTable((resultRowCollections.GetValue(0) as IBlueRowCollection[])[0]);
-               myFireEvent(new DataPacket(myLastTableVersion, new BlueTable(myLastTableVersion.schema), new BlueTable(myLastTableVersion.schema)));
-            }
-            else
-            {
-               // BlueEventLog.logAssert(ops[0] == (byte)StreamOpCode.multipleRemoveAdd, "Playback stream got unexpected open code " + ops[0].ToString() + " from file " + myFileName);
-               // BlueEventLog.logAssert(myLastTableVersion != null, "Playback stream for file "+myFileName + "got a non-create before a create");
-               IBlueRowCollection[] deletedAndAdded = resultRowCollections.GetValue(0) as IBlueRowCollection[];
-               BlueDataUtility.absorbChanges(myLastTableVersion, deletedAndAdded[0], deletedAndAdded[1]);
-               myFireEvent(new DataPacket(myLastTableVersion, deletedAndAdded[1], deletedAndAdded[0]));
-            }
-            
-            myReadNextPacketTime();
+      public void doWork() {
+         if (_all_files_read)
+            return;
+         if (current_file_all_replayed) {
+            _load_next_file();
          }
+         PlaybackPacket curr_packet = _locate_packet_to_replay();
+         if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+            byte[] ops;
+            Array row_array;
+            // what a stupid interface!
+            _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+            _fire_packet(ops, row_array);
+            curr_packet.replay_timestamp = PersistorStream.epoch_now();
+         } else {
+            PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+            // this is the time we need to fire (replay) this packet
+            long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+            long now_epoch = PersistorStream.epoch_now();
+            // fire when we are past the 
+            if (now_epoch >= time_to_fire) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+               _fire_packet(ops, row_array);
+               curr_packet.replay_timestamp = PersistorStream.epoch_now();
+            }
+         }
+      }
 
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
+
+         }
+      }
+
+      private void _load_next_file() {
+         var new_idx = _current_fs.Item1 + 1;
+         if (new_idx >= _file_names.Count()) {
+            _all_files_read = true;
+            bts.Log.Gen.info("All files read");
+            return;
+         }
+         var prev_fs = _current_fs.Item2;
+         if (prev_fs != null)
+            prev_fs.Close();
+         var new_filename = _file_names.ElementAt(new_idx);
+         bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+         var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+         _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+         _load_packets_from_current_file();
       }
 
       #endregion

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -0,0 +1,115 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using bts.collections;
+using System.Reflection.Metadata;
+
+
+namespace BCG {
+   public class SnapshotStream : ViewStream {
+
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      public DateTime _snapshot_time;
+      public string _snapshot_file;
+      public IEnumerable<PlaybackPacket> _packets;
+
+
+      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+         string token_directory_name = $"{directory_name}\\{token}";
+         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+         _packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
+      }
+
+      private void _absorb_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+            } else {
+               bts.Log.assert(false, $"Operator: {op} is not expected");
+            }
+         }
+      }
+
+      protected override void myOnLastSubscriber() {
+      }
+
+      // public event SnapshotStream.Progress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
+         }
+         return null;
+      }
+
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
+
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -0,0 +1,145 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Globalization;
+using System.IO;
+
+
+namespace BCG {
+   // each stream op in the replay file is reprsented by a packet here
+   public class PlaybackPacket {
+      public readonly int idx;
+      public readonly long timestamp;
+      public readonly byte[] data;
+      public long replay_timestamp = 0; // non zero it has been replayed
+      public long offset_from_previous = 0;
+      public PlaybackPacket(int the_idx, long the_timestamp, byte[] the_data) {
+         this.idx = the_idx;
+         this.timestamp = the_timestamp;
+         this.data = the_data;
+      }
+      public bool replayed {
+         get { return replay_timestamp != 0; }
+      }
+   }
+
+   public enum PlaybackType {
+      Streaming = 0,
+      Snapshot
+   }
+   public class SnapshotData {
+      public DateTime snapshot_time;
+      public string snapshot_file;
+      public SnapshotData(DateTime snapshot_time_arg, string snapshot_file_arg) {
+         snapshot_time = snapshot_time_arg;
+         snapshot_file = snapshot_file_arg;
+      }
+      public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
+   }
+   public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            string next_fn = file_names[i + 1];
+            DateTime f_dt = PersistorStream.parse_datetime_from_filename(fn);
+            DateTime next_f_dt = PersistorStream.parse_datetime_from_filename(next_fn);
+            if (snapshot_time >= f_dt && snapshot_time <= next_f_dt) {
+               return fn;
+            }
+         }
+         return String.Empty;
+      }
+      public static DateTime parse_datetime_from_filename(string s) {
+         // allbooks_1697054438_2023-10-11_16-00-38
+         string[] toks = s.Split("_");
+         bts.Log.assert(toks.Length == 4);
+         string dt = $"{toks[2]}_{toks[3]}";
+         return parse_datetime(dt);
+      }
+
+      // this is used to parse the string passed the datetime texts passed to cons of the stream 
+      public static DateTime parse_datetime(string s) {
+         // for whatever reason "yyyy-MM-dd_HH-mm-ss" is not parsable using DateTime.ParseExact (it only gets
+         // the Date not the time), "yyyy-MM-dd_HH:mm:ss" is parsable
+         // however a file name can not have : in its name, therefore we need to parse "yyyy-MM-dd_HH-mm-ss"
+         DateTime rv = DateTime.ParseExact(s, PersistorStream.DateTime_format, CultureInfo.InvariantCulture);
+         string[] toks = s.Split("_");
+         string time_str = toks[1];
+         string[] time_toks = time_str.Split("-");
+         bts.Log.assert(time_toks.Length == 3);
+         int hh = Int32.Parse(time_toks[0]);
+         int mm = Int32.Parse(time_toks[1]);
+         int ss = Int32.Parse(time_toks[2]);
+         rv = rv.Date;
+         rv = rv.AddHours(hh);
+         rv = rv.AddMinutes(mm);
+         rv = rv.AddSeconds(ss);
+         return rv;
+      }
+
+      public static string DateTime_format {
+         get {
+            return "yyyy-MM-dd_HH-mm-ss";
+         }
+      }
+
+      public static long dt_to_epoch(DateTime dt) {
+         DateTimeOffset dtos = new DateTimeOffset(dt, TimeSpan.Zero);
+         return dtos.ToUnixTimeSeconds();
+      }
+      public static long epoch_now() {
+         return DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+      }
+      public static DateTime epoch_to_dt(long epoch_seconds) {
+         return DateTimeOffset.FromUnixTimeSeconds(epoch_seconds).DateTime;
+      }
+
+      public static PlaybackPacket find_packet_at_time(IEnumerable<PlaybackPacket> packets, DateTime snapshot_time) {
+         log_file_ts(packets, snapshot_time);
+         long snapshot_time_epoch = PersistorStream.dt_to_epoch(snapshot_time);
+         for (int i = 0; i < packets.Count() - 1; i++) {
+            PlaybackPacket curr_p = packets.ElementAt(i);
+            PlaybackPacket next_p = packets.ElementAt(i + 1);
+            if (curr_p.timestamp <= snapshot_time_epoch && next_p.timestamp >= snapshot_time_epoch) {
+               return curr_p;
+            }
+         }
+         // if for some reason it cant find it, send the last packet
+         return packets.Last();
+      }
+
+      private static void log_file_ts(IEnumerable<PlaybackPacket> packets, DateTime snapshot_time) {
+         string big_log_msg = String.Empty;
+         foreach (var p in packets) {
+            big_log_msg += $"packet idx: {p.idx} epoch_ts: {p.timestamp} ts: {epoch_to_dt(p.timestamp)} snapshot_time: {snapshot_time}";
+            big_log_msg += "\n";
+         }
+         bts.Log.Gen.info($"Packets: {big_log_msg}");
+      }
+
+      public static IEnumerable<PlaybackPacket> load_packets_from_file(FileStream fs) {
+         BinaryReader br = new BinaryReader(fs);
+         List<PlaybackPacket> rv = new List<PlaybackPacket>();
+         while (fs.Position < fs.Length) {
+            try {
+               long ts = br.ReadInt64();
+               int sz = br.ReadInt32();
+               byte[] data = br.ReadBytes(sz);
+               var pb = new PlaybackPacket(rv.Count, ts, data);
+               rv.Add(pb);
+            } catch (Exception ex) {
+               bts.Log.Gen.error($"ex: {ex}");
+               break;
+            }
+         }
+         // setting offset field
+         for (int i = 0; i < rv.Count; i++) {
+            if (i == 0)
+               continue;
+            rv[i].offset_from_previous = rv[i].timestamp - rv[i - 1].timestamp;
+         }
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -8,6 +8,7 @@ using mbts.spark;
 
 namespace BCG
 {
+   [DataStreamUserRespresentation("fastJoin")]
    public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
    {
       private readonly NonIncJoinSpec[] _join_specs;
@@ -23,7 +24,7 @@ namespace BCG
 
       public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
                               ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[] 
+         : this(new Triplet<ViewStream, string, string>[]
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
@@ -31,7 +32,7 @@ namespace BCG
       { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
       { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
@@ -102,8 +103,8 @@ namespace BCG
          {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema (cis);
-            _output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
+               d.constribute_cols_to_schema(cis);
+            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs

@@ -0,0 +1,48 @@
+
+
+using System;
+
+namespace BCG {
+   // Keeps adding changing rows to the stream, hence preserving the history
+   // and creating a time series, for example this can be used to make bbo updates
+   // into a time series of prices/qty 
+   public class DebugStream : ViewStream {
+
+      private ViewStream _stream;
+      private IBlueRowCollection _last_recv_rows = null;
+
+      public DebugStream(ViewStream stream) {
+         _stream = stream;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _stream.onChange += new ChangeDelegate(_onstream_change);
+      }
+      protected override void myOnLastSubscriber() {
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
+      }
+
+      private void _onstream_change(object o) {
+         DataPacket dp = o as DataPacket;
+         BlueDataUtility.validate_table_schema(dp.newRows);
+         BlueDataUtility.validate_table_schema(dp.addedRows);
+         if (_last_recv_rows == null) {
+            _last_recv_rows = dp.newRows;
+         } else {
+            //schema 
+            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.newRows.schema));
+            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.addedRows.schema));
+            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.deletedRows.schema));
+            foreach (IBlueRow r in dp.deletedRows) {
+               bts.Log.assert(_last_recv_rows.containsID(r.get<int>("id")));
+            }
+            foreach (IBlueRow r in dp.addedRows) {
+               bts.Log.assert(dp.newRows.containsID(r.get<int>("id")));
+            }
+            _last_recv_rows = dp.newRows;
+         }
+         myFireEvent(o);
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -0,0 +1,169 @@
+using Google.Protobuf.WellKnownTypes;
+
+using System;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+   // Keeps adding changing rows to the stream, hence preserving the history
+   // and creating a time series, for example this can be used to make bbo updates
+   // into a time series of prices/qty 
+   public class TimeSeriesStream : ViewStream
+   {
+      private BlueTable _table;
+      private ViewStream _stream;
+      private int _id = 0;
+
+      public TimeSeriesStream(ViewStream stream)
+      {
+         _stream = stream;
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _stream.onChange += new ChangeDelegate(_onstream_change);
+      }
+      protected override void myOnLastSubscriber()
+      {
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
+      }
+
+
+      private void _onstream_change(object o)
+      {
+         DataPacket dp = o as DataPacket;
+         DateTime now = DateTime.Now;
+         if (_table == null)
+         {
+            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            _table = new BlueTable(schema);
+         }
+         BlueTable added_rows = new BlueTable(_table.schema);
+         foreach (IBlueRow r in dp.addedRows)
+         {
+            MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
+            ar.setValue("timestamp", now);
+            ar.setValue("id", _id++);
+            added_rows.addRow(ar);
+         }
+         _table.addRows(added_rows);
+         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+
+      }
+   }
+
+   public class TimeBucketStream : ViewStream, IStreamMapper
+   {
+      private ViewStream _src;
+      private readonly string _time_stamp_col;
+      private IBlueTableSchema _schema;
+      private int _bucket_size_seconds;
+
+      public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
+      {
+         _time_stamp_col = time_stamp_col;
+         _bucket_size_seconds = bucket_size_seconds;
+         var vs = new MapStream(input, this);
+         mySetRealSource(vs);
+      }
+
+      #region IStreamMapper Members
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
+      {
+         IBlueTableSchema rv = input_schema;
+         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
+         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
+         return rv;
+
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime ts = row.get<DateTime>(_time_stamp_col);
+         TimeSpan time_of_day = (ts - ts.Date);
+         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
+         double buckets = seconds / _bucket_size_seconds;
+         int bucket_number = Convert.ToInt32(Math.Floor(buckets));
+         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
+         rv.setValue("bucket_number", bucket_number);
+         return rv;
+      }
+
+      #endregion
+   }
+
+   // public class ThrottleStream : ViewStream, IWorker
+   // {
+   //    private ViewStreamDatabase _vsd;
+   //    private ViewStream _stream;
+   //    private int _throttle_ms;
+   //    private TimeSpan _min_fire_period;
+   //    private DateTime _last_Fire_time = DateTime.MinValue;
+   //    private DataPacket _unfired_dp;
+   //    public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
+   //    {
+   //       _vsd = vsd;
+   //       _stream = base_stream;
+   //       _throttle_ms = throttle_ms;
+   //       _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
+   //    }
+
+   //    protected override void myOnFirstSubscriber()
+   //    {
+   //       _vsd.addWorker(this);
+   //       _stream.onChange += new ChangeDelegate(_onstream_change);
+   //    }
+   //    protected override void myOnLastSubscriber()
+   //    {
+   //       _vsd.removeWorker(this);
+   //       _stream.onChange -= new ChangeDelegate(_onstream_change);
+   //    }
+
+   //    private void _onstream_change(object obj)
+   //    {
+   //       DataPacket dp = obj as DataPacket;
+   //       if (DateTime.Now - _last_Fire_time >= _min_fire_period)
+   //       {
+   //          // ok to fire here
+   //       }
+   //       else
+   //       {
+   //          _last_cached_state = dp.newRows;
+   //       }
+   //       _possibly_fire(dp.newRows);
+   //    }
+
+   //    public void doWork()
+   //    {
+   //       if (_unfired_dp != null)
+   //       {
+
+   //       }
+   //    }
+
+   //    private void _possibly_fire()
+   //    {
+   //       if (_unfired_dp != null)
+   //       {
+   //          if (DateTime.Now - _last_fire_time > _min_fire_period)
+   //          {
+   //             DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
+   //             _unfired_dp = null;
+   //             _last_Fire_time = DateTime.Now;
+   //             myFireEvent(dp);
+   //          }
+   //       }
+   //    }
+
+   // }
+
+
+}
+
+
+

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -6,7 +6,7 @@ using BCG.Utilities;
 
 namespace BCG
 {
-   [DataStreamUserRespresentation("csvSaver")]
+   [DataStreamUserRespresentation("CSVSaver")]
    public class CSVSaverStream : ViewStream, ISideEffectStream
    {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -0,0 +1,143 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+
+namespace BCG
+{
+   /// <summary>
+   /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
+   /// </summary>
+   public enum DateConversion
+   {
+      ConvertDoubleToDateTime = 0,
+      ChopTime,
+      UTCToCurrent,
+      ConvertStringToDateTime,
+      ConvertDateTimeToMilliseconds,
+      SecondsOfDay
+   }
+
+   [DataStreamUserRespresentation("dateStandardize")]
+   public class DateStandardizationStream : ViewStream, IStreamMapper
+   {
+      private IBlueTableSchema mySchema;
+      private DateConversion myConversion;
+      private string myConversionColName;
+      private readonly Dictionary<DateConversion, Type> _date_conversion_to_type_map = new Dictionary<DateConversion, Type>()
+      {
+         {DateConversion.ConvertDoubleToDateTime      , typeof(DateTime)},
+         {DateConversion.ChopTime                     , typeof(DateTime)},
+         {DateConversion.UTCToCurrent                 , typeof(DateTime)},
+         {DateConversion.ConvertStringToDateTime      , typeof(DateTime)},
+         {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
+         {DateConversion.SecondsOfDay                 , typeof(Double)}
+      };
+
+
+      private static DateConversion _parse(string s)
+      {
+         DateConversion rv = DateConversion.ChopTime;
+         Enum.TryParse<DateConversion>(s, false, out rv);
+         return rv;
+      }
+      /// <summary>
+      /// for DSQA
+      /// </summary>
+      public DateStandardizationStream(ViewStream vs, string colName, string conversion)
+         : this(vs, colName, _parse(conversion))
+      { }
+
+      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
+      {
+         myConversionColName = colName;
+         myConversion = conversion;
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (ColumnInfo c in inputSchema.columnInfo)
+            {
+               if (c.colName == myConversionColName)
+               {
+                  Type convType;
+                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
+                  {
+                     //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
+                     convType = typeof(DateTime);
+                  }
+                  cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
+               }
+               else
+                  cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
+            }
+            mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
+      }
+
+      #endregion
+
+      public static object mapDateTime(object objIn, DateConversion conv)
+      {
+         if (objIn == DBNull.Value || objIn == null)
+         {
+            return DBNull.Value;
+         }
+         else
+         {
+            if (conv == DateConversion.ChopTime)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               return dtIn.Date;
+            }
+            else if (conv == DateConversion.ConvertDoubleToDateTime)
+            {
+               double dtIn = (double)objIn;
+               return DateTime.FromOADate(dtIn);
+            }
+            else if (conv == DateConversion.UTCToCurrent)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               return dtIn.ToLocalTime();
+            }
+            else if (conv == DateConversion.ConvertStringToDateTime)
+            {
+               return DateTime.Parse((string)objIn);
+            }
+            else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               return dtIn.TimeOfDay.TotalMilliseconds;
+            }
+            else if (conv == DateConversion.SecondsOfDay)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               DateTime start_of_day = dtIn.Date;
+               TimeSpan day_period = dtIn - start_of_day;
+               return day_period.TotalSeconds;
+            }
+            else
+            {
+               bts.Log.Gen.info("incorrect type");
+               return null;
+            }
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Collections;
+
+namespace BCG
+{
+   [DataStreamUserRepresentation("timeBucket")]
+   public class TimeBucketStream : ViewStream, IStreamMapper
+   {
+      private ViewStream _src;
+      private readonly string _time_stamp_col;
+      private IBlueTableSchema _schema;
+      private uint _bucket_size_seconds;
+
+      public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
+      {
+         _time_stamp_col = time_stamp_col;
+         _bucket_size_seconds = bucket_size_seconds;
+      }
+
+      #region IStreamMapper Members
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
+      {
+         IBlueTableSchema rv = input_schema;
+         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
+         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
+         return rv;
+
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime ts = row.get<DateTime>(_time_stamp_col);
+         TimeSpan time_of_day = (ts - ts.Date)
+         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
+         int buckets = seconds / _bucket_size_seconds;
+         int bucket_number = Math.Floor(buckets);
+         MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
+         rv.setValue("bucket_number", bucket_number);
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs

@@ -1,10 +1,9 @@
 using System;
+
 using BCG.Utilities.General;
 
-namespace BCG
-{
-	public abstract class TimeUnitStream : ViewStream, IWorker
-	{
+namespace BCG {
+   public abstract class TimeUnitStream : ViewStream, IWorker {
       protected TimeSpan myOffset;
       protected DateTime myConstructionTime;
       protected int mySizeOfStream;
@@ -13,87 +12,80 @@ namespace BCG
       protected ViewStreamDatabase myDb;
       protected int myId = 0;
 
-      public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
-		{
+      public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
          myConstructionTime = DateTime.Now;
          myZeroTime = zeroTime;
          myOffset = offSet;
          mySizeOfStream = sizeOfStream;
          myDb = db;
-         myCollection = new CircularRowCollection (
-            new BlueTableSchema ("Time",
-            new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))), 
+         myCollection = new CircularRowCollection(
+            new BlueTableSchema("Time",
+            new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
             mySizeOfStream);
-		}
+      }
 
       #region ViewStream overriden methods
 
-      protected override void myOnFirstSubscriber()
-      {
-         myDb.addWorker (this);
-         myInit ();
+      protected override void myOnFirstSubscriber() {
+         myDb.addWorker(this);
+         myInit();
       }
 
-      protected override void myOnLastSubscriber()
-      {
-         myDb.removeWorker (this);
+      protected override void myOnLastSubscriber() {
+         myDb.removeWorker(this);
       }
 
       #endregion
 
-      protected virtual void myInit ()
-      {
+      protected virtual void myInit() {
          int periods = (int)
-            (DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
-            
+            (DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
 
-         DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) : 
-            myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
+
+         DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
+            myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
 
 
          if (myGetLatestRow() == null ||
-            !myGetLatestRow()["time"].Equals(dt))
-         {
-            IBlueRow ar = new BlueRow (myCollection.schema,
+            !myGetLatestRow()["time"].Equals(dt)) {
+            IBlueRow ar = new BlueRow(myCollection.schema,
                myId++, dt);
-            IBlueRow dr = myCollection.addRowWithReturn (ar);
-            myFireEvent (new DataPacket (myCollection,
-               myConvertToCollection (ar),
-               myConvertToCollection (dr)));
+            IBlueRow dr = myCollection.addRowWithReturn(ar);
+            if (n_updates <= 3000)
+               myFireEvent(new DataPacket(myCollection,
+                  myConvertToCollection(ar),
+                  myConvertToCollection(dr)));
          }
       }
 
       #region IWorker methods
 
-      public void doWork ()
-      {
-         myInit ();
+      public void doWork() {
+         myInit();
       }
 
       #endregion
 
-      protected abstract DateTime myRoundTime (DateTime dt);
-         
+      protected abstract DateTime myRoundTime(DateTime dt);
+
 
       #region Private Helper methods
 
-      private IBlueRow myGetLatestRow ()
-      {
+      private IBlueRow myGetLatestRow() {
          return myCollection.Count == 0 ?
             null :
-            BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
+            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
       }
 
-      private IBlueRowCollection myConvertToCollection (IBlueRow r)
-      {
-         BlueTable retVal = new BlueTable (myCollection.schema);
+      private IBlueRowCollection myConvertToCollection(IBlueRow r) {
+         BlueTable retVal = new BlueTable(myCollection.schema);
          if (r != null)
-            retVal.addRow (r);
+            retVal.addRow(r);
          return retVal;
       }
 
       #endregion
 
-	}
+   }
 
 }

src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Xml.Serialization;
 
 using BCG.Utilities;
 
@@ -53,7 +55,12 @@ namespace BCG {
       }
 
       public virtual void createTable(IBlueRowCollection newTable) {
-         bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
+         string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
+         foreach (IBlueRow r in newTable) {
+            log_msg += $"id: {r["id"]}";
+         }
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(newTable.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = newTable.schema.tableName.ToLower();
@@ -71,6 +78,9 @@ namespace BCG {
       /// </summary>
       /// <param name="r"></param>
       public virtual void addRow(IBlueRow r) {
+         string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(r.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = r.schema.tableName.ToLower();
@@ -86,6 +96,9 @@ namespace BCG {
       /// </summary>
       /// <param name="r"></param>
       public virtual void deleteRow(IBlueRow r) {
+         string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(r.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = r.schema.tableName.ToLower();
@@ -101,6 +114,9 @@ namespace BCG {
       /// </summary>
       /// <param name="r"></param>
       public virtual void changeRow(IBlueRow r) {
+         string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(r.schema);
          myCheckTransaction();
          IPubTablePublisher table_publisher = null;
          var name_key = r.schema.tableName.ToLower();
@@ -118,7 +134,16 @@ namespace BCG {
       public virtual void makeBatchChanges(IBlueRowCollection newRows,
                                              IBlueRowCollection deletedRows,
                                              IBlueRowCollection addedRows) {
-         myCheckTransaction();
+         string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
+         foreach (IBlueRow r in deletedRows)
+            log_msg += $"id: {r["id"]} ";
+         log_msg += "addedRows: ";
+         foreach (IBlueRow r in addedRows)
+            log_msg += $"id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(newRows.schema);
+         _check_schema_consistency(deletedRows.schema);
+         _check_schema_consistency(addedRows.schema);
          IPubTablePublisher table_publisher = null;
          var name_key = newRows.schema.tableName.ToLower();
          if (!_publisher_map.TryGetValue(name_key, out table_publisher))
@@ -144,8 +169,16 @@ namespace BCG {
       }
 
       protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
+         bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
          return new PubTablePublisher(tag, _publisher_core);
       }
 
+      private void _check_schema_consistency(IBlueTableSchema schema_arg) {
+         IBlueTableSchema schema;
+         if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
+            bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
+         }
+      }
+
    }
 }

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -14,7 +14,7 @@ namespace BCG {
       /// is not uniquely chosen)
       public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
          myName = name;
-         myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
+         myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
          myBroker = br;
          myFirstTime = true;
          myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -12,6 +12,7 @@ using BCG.Utilities.General;
 using System.Linq.Expressions;
 //using mbts.spark;
 using mbts;
+using BCG.Extensions;
 
 namespace BCG
 {
@@ -42,6 +43,20 @@ namespace BCG
       {
          return get_row_value(r, col_no, err_value);
       }
+      public static IEnumerable<T> get_covalues<T>(this IBlueRowCollection rows, string col)
+      {
+         List<T> rv = new List<T>();
+         foreach (IBlueRow r in rows)
+            rv.Add(r.get<T>(col));
+         return rv;
+      }
+      public static IEnumerable<T> get_covalues<T>(IEnumerable<IBlueRow> rows, string col)
+      {
+         List<T> rv = new List<T>();
+         foreach (IBlueRow r in rows)
+            rv.Add(r.get<T>(col));
+         return rv;
+      }
 
       public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func)
       {
@@ -85,6 +100,30 @@ namespace BCG
          }
       }
 
+      public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp)
+      {
+         IBlueRowCollection added_rows = new_rows;
+         IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows;
+         return new DataPacket(new_rows, added_rows, deleted_rows);
+      }
+
+      public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp)
+      {
+         if (last_fired_dp == null)
+         {
+            IBlueRowCollection added_rows = new_rows;
+            IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema);
+            return new DataPacket(new_rows, added_rows, deleted_rows);
+         }
+         else
+         {
+            IBlueRowCollection added_rows = null;
+            IBlueRowCollection deleted_rows = null;
+            compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows);
+            return new DataPacket(new_rows, added_rows, deleted_rows);
+         }
+      }
+
       public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows,
                                                              bool handle_datetime_min = false,
                                                              bool handle_double_nan = false)
@@ -113,16 +152,22 @@ namespace BCG
       #endregion
 
       public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width)
+      {
+         var rv = get_printed_rows(table, get_col_width);
+         Console.WriteLine(rv);
+      }
+
+      public static string get_printed_rows(this IBlueRowCollection table, Func<string, int> get_col_width)
       {
          // Print column headers
-         Console.WriteLine($"{table.schema.tableName}______________________________________");
+         string rv = String.Empty;// = $"{table.schema.tableName}______________________________________";
          foreach (var col_name in table.schema.columnNames)
          {
             int col_width = get_col_width(col_name);
-            Console.Write($"{col_name.PadRight(col_width)}  ");
+            rv += $"{col_name.PadRight(col_width)}  ";
          }
-         Console.WriteLine();
-         Console.WriteLine("______________________________________");
+         rv += "\n";
+         //rv += "______________________________________";
 
          // Print table data
          foreach (IBlueRow row in table)
@@ -134,11 +179,12 @@ namespace BCG
                string row_val_str = row_val.ToString();
                if (row_val != null)
                {
-                  Console.Write($"{row_val_str.PadRight(col_width)} ");
+                  rv += $"{row_val_str.PadRight(col_width)} ";
                }
             }
-            Console.WriteLine();
+            rv += "\n";
          }
+         return rv;
       }
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
       {
@@ -919,6 +965,13 @@ namespace BCG
          compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
       }
 
+      public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
+                                                      out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows)
+      {
+         int[] col_indices = new int[] { new_rows.schema["id"].colNo };
+         compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
+      }
+
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows)
       {
@@ -1294,7 +1347,7 @@ namespace BCG
                foreach (ColumnInfo ci1 in s1.columnInfo)
                {
                   ColumnInfo ci2 = s2[ci1.colNo];
-                  if (ci1.colName != ci2.colName || ci1.colType != ci2.colType)
+                  if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo)
                   {
                      result = false;
                      break;
@@ -1308,6 +1361,25 @@ namespace BCG
          return result;
       }
 
+      public static void validate_row_schema(IBlueRow r)
+      {
+         foreach (var ci in r.schema.columnInfo)
+         {
+            object rval = r[ci.colName];
+            if (rval == null)
+               continue;
+            bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType));
+         }
+      }
+
+      public static void validate_table_schema(IBlueRowCollection rows)
+      {
+         foreach (IBlueRow r in rows)
+         {
+            validate_row_schema(r);
+         }
+      }
+
       public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals)
       {
          IBlueRowCollection dels = findMatchingRows(table, cols, vals);

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -5,14 +5,12 @@ using System.Data;
 //using BCG.Utilities;
 using System.Linq;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Anologue of the dotNet DataTable.
    /// </summary>
    [Serializable]
-   public class BlueTable : IBlueRowCollection
-   {
+   public class BlueTable : IBlueRowCollection {
       /// <summary>
       /// Hashtable indexed on "id" to speed up removes
       /// </summary>
@@ -29,22 +27,19 @@ namespace BCG
       /// all the data over
       /// </summary>
       /// <param name="table"></param>
-      public BlueTable(DataTable table)
-      {
+      public BlueTable(DataTable table) {
          ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
          int i = 0;
          foreach (DataColumn col in table.Columns)
             cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType);
          mySchema = new BlueTableSchema(cis, table.TableName);
-         foreach (DataRow row in table.Rows)
-         {
+         foreach (DataRow row in table.Rows) {
             IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
             myIdIndex[newRow["id"]] = newRow;
          }
       }
 
-      public BlueTable(IBlueRowCollection table)
-      {
+      public BlueTable(IBlueRowCollection table) {
          mySchema = table.schema;
          foreach (IBlueRow row in table)
             addRow(row);
@@ -52,12 +47,10 @@ namespace BCG
 
       public BlueTable(string tableName,
           IEnumerable objects,
-          params CMap[] columnMappers)
-      {
+          params CMap[] columnMappers) {
          // extract schema from mappers
          ColumnInfo[] cols = new ColumnInfo[columnMappers.Length];
-         for (int i = 0; i < columnMappers.Length; i++)
-         {
+         for (int i = 0; i < columnMappers.Length; i++) {
             cols[i] = new ColumnInfo(
                 columnMappers[i].getColumnName(), i,
                 columnMappers[i].getColumnType());
@@ -66,8 +59,7 @@ namespace BCG
 
          // load rows
          int rowNumber = 0;
-         foreach (object rowObject in objects)
-         {
+         foreach (object rowObject in objects) {
             object[] values = new object[columnMappers.Length];
             for (int cn = 0; cn < columnMappers.Length; cn++)
                values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject);
@@ -87,10 +79,8 @@ namespace BCG
       /// <summary>
       /// Returns the row by the index, interface property
       /// </summary>
-      public virtual IBlueRow this[int id]
-      {
-         get
-         {
+      public virtual IBlueRow this[int id] {
+         get {
             object row = myIdIndex[id];
             if (row == null)
                throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
@@ -105,8 +95,7 @@ namespace BCG
 
       public virtual bool containsID(int id) { return myIdIndex[id] != null; }
 
-      public virtual bool tryGetValue(int id, out IBlueRow row)
-      {
+      public virtual bool tryGetValue(int id, out IBlueRow row) {
          row = (IBlueRow)myIdIndex[id];
          return (row != null);
       }
@@ -114,8 +103,7 @@ namespace BCG
       #endregion
 
       #region BlueTable members
-      public virtual void clear()
-      {
+      public virtual void clear() {
          myIdIndex.Clear();
       }
 
@@ -123,8 +111,7 @@ namespace BCG
       /// If a row with the same id exists, it replaces it
       /// </summary>
       /// <param name="r"></param>
-      public virtual void forceAddRow(IBlueRow r)
-      {
+      public virtual void forceAddRow(IBlueRow r) {
          myIdIndex[r["id"]] = r;
       }
 
@@ -132,43 +119,41 @@ namespace BCG
       /// Adds a new row to the table. 
       /// </summary>
       /// <param name="row"></param>
-      public virtual void addRow(IBlueRow row)
-      {
+      public virtual void addRow(IBlueRow row) {
          object id = row["id"];
 
          if (id == null || id == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
-         try
-         {
+         try {
             myIdIndex.Add(id, row);
-         }
-         catch (ArgumentException)
-         {
-            throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed");
+         } catch (ArgumentException) {
+            throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}");
          }
 
       }
 
-      public virtual void addRow(DataRow row)
-      {
+      public virtual void addRow(DataRow row) {
          object val = row["id"];
 
          if (val == null || val == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
-         if (myIdIndex[val] != null)
-            throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed");
+         if (myIdIndex[val] != null) {
+            string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed";
+            // temp TODOds remove it
+            bts.Log.assert(false, err_msg);
+            throw new BlueDataException(err_msg);
+
+         }
          IBlueRow newRow = new BlueRow(mySchema, row);
          myIdIndex[val] = newRow;
       }
 
-      public virtual void addRows(IBlueRowCollection rows)
-      {
+      public virtual void addRows(IBlueRowCollection rows) {
          foreach (IBlueRow nr in rows)
             addRow(nr);
       }
 
-      public virtual void addRows(IEnumerable rows)
-      {
+      public virtual void addRows(IEnumerable rows) {
          foreach (IBlueRow nr in rows)
             addRow(nr);
       }
@@ -185,37 +170,32 @@ namespace BCG
       /// Removes a row specified by the id. The index helps in making this operation O(1)
       /// </summary>
       /// <param name="id"></param>
-      public virtual bool deleteRow(int id)
-      {
+      public virtual bool deleteRow(int id) {
          if (myIdIndex[id] == null)
             return false;
          myIdIndex.Remove(id);
          return true;
       }
 
-      public virtual bool deleteRows(IBlueRowCollection rows)
-      {
+      public virtual bool deleteRows(IBlueRowCollection rows) {
          bool retVal = true;
          foreach (IBlueRow r in rows)
             retVal &= deleteRow(r);
          return retVal;
       }
 
-      public virtual bool deleteRows(IEnumerable rows)
-      {
+      public virtual bool deleteRows(IEnumerable rows) {
          bool retVal = true;
          foreach (IBlueRow r in rows)
             retVal &= deleteRow(r);
          return retVal;
       }
 
-      public virtual void changeRow(IBlueRow r)
-      {
+      public virtual void changeRow(IBlueRow r) {
          myIdIndex[r["id"]] = r;
       }
 
-      public virtual void changeRows(IBlueRowCollection rows)
-      {
+      public virtual void changeRows(IBlueRowCollection rows) {
          foreach (IBlueRow r in rows)
             changeRow(r);
       }

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -1,316 +1,316 @@
-// using System;
-// using System.Collections;
-// using System.Data;
-// using BCG.Utilities.General;
+using System;
+using System.Collections;
+using System.Data;
+using BCG.Utilities.General;
 
-// namespace BCG
-// {
-// 	[Serializable]
-//    public class DataPacket : ICloneable, IDataPacket
-//    {      
-//       public class BlueRowIdComparator
-//       {
-//          private string[] myIdColumns = new string[]{"id"};
+namespace BCG
+{
+	[Serializable]
+   public class DataPacket : ICloneable, IDataPacket
+   {      
+      public class BlueRowIdComparator
+      {
+         private string[] myIdColumns = new string[]{"id"};
 
-//          public BlueRowIdComparator(string[] idCols)
-//          {
-//             if ( idCols.Length != 0 ) 
-//             {
-//                myIdColumns = idCols;
-//             }
-//          }
+         public BlueRowIdComparator(string[] idCols)
+         {
+            if ( idCols.Length != 0 ) 
+            {
+               myIdColumns = idCols;
+            }
+         }
+                    
+         public  bool blueRowIdEqualEvaluator(object a, object b)
+         {
+            if ( a == null || b == null )
+            {
+               return false;
+            }
 
-//          public  bool blueRowIdEqualEvaluator(object a, object b)
-//          {
-//             if ( a == null || b == null )
-//             {
-//                return false;
-//             }
+            if ( !(a is IBlueRow) || !(b is IBlueRow) )
+            {
+               return false;
+            }
 
-//             if ( !(a is IBlueRow) || !(b is IBlueRow) )
-//             {
-//                return false;
-//             }
+            IBlueRow lhs = (IBlueRow)a;
+            IBlueRow rhs = (IBlueRow)b;
+                    
+            bool result = true;
+            foreach(string idColumn in myIdColumns)
+            {
+               result = result && lhs[idColumn].Equals(rhs[idColumn]);
+               if ( !result )
+                  break;
+            }
 
-//             IBlueRow lhs = (IBlueRow)a;
-//             IBlueRow rhs = (IBlueRow)b;
+            return result;            
+         }
+      }
 
-//             bool result = true;
-//             foreach(string idColumn in myIdColumns)
-//             {
-//                result = result && lhs[idColumn].Equals(rhs[idColumn]);
-//                if ( !result )
-//                   break;
-//             }
+      public IBlueRowCollection newRows, addedRows, deletedRows;
+      public DataPacket () {}
+      public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
+      {
+         this.newRows = newRows;
+         this.addedRows = addedRows;
+         this.deletedRows = deletedRows;
+      }
 
-//             return result;            
-//          }
-//       }
+      public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
+      {
+         new_rows = newRows;
+         added_rows = addedRows;
+         deleted_rows = deletedRows;
+      }
 
-//       public IBlueRowCollection newRows, addedRows, deletedRows;
-//       public DataPacket () {}
-//       public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
-//       {
-//          this.newRows = newRows;
-//          this.addedRows = addedRows;
-//          this.deletedRows = deletedRows;
-//       }
+      public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
+      {                 
+         BlueTable truly = new BlueTable(addedRows.schema);
+         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-//       public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
-//       {
-//          new_rows = newRows;
-//          added_rows = addedRows;
-//          deleted_rows = deletedRows;
-//       }
+         CollectionHelpers.EqualDelegate ed = 
+            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-//       public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
-//       {                 
-//          BlueTable truly = new BlueTable(addedRows.schema);
-//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+         ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+           
+         foreach(IBlueRow r in ta)
+         {
+            truly.addRow(r);
+         }
+           
+         return truly;
+                
+      }
 
-//          CollectionHelpers.EqualDelegate ed = 
-//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+      public virtual IBlueRowCollection getTrulyAddedRows ()
+      {
+         BlueTable retVal = new BlueTable (schema);
+         foreach (IBlueRow r in addedRows)
+            if (!deletedRows.containsID ((int)r["id"]))
+               retVal.addRow (r);
+         return retVal;
+      }
 
-//          ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+      public virtual IBlueRowCollection getTrulyDeletedRows ()
+      {
+         BlueTable retVal = new BlueTable (schema);
+         foreach (IBlueRow r in deletedRows)
+            if (!addedRows.containsID ((int)r["id"]))
+               retVal.addRow (r);
+         return retVal;
+      }
 
-//          foreach(IBlueRow r in ta)
-//          {
-//             truly.addRow(r);
-//          }
+      public virtual IBlueRowCollection getTrulyChangedRows ()
+      {
+         BlueTable retVal = new BlueTable (schema);
+         foreach (IBlueRow r in addedRows)
+            if (deletedRows.containsID ((int)r["id"]))
+               retVal.addRow (r);
+         return retVal;
+      }
 
-//          return truly;
+      
+      public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
+      {        
+         BlueTable truly = new BlueTable(deletedRows.schema);
+         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-//       }
+         CollectionHelpers.EqualDelegate ed = 
+            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-//       public virtual IBlueRowCollection getTrulyAddedRows ()
-//       {
-//          BlueTable retVal = new BlueTable (schema);
-//          foreach (IBlueRow r in addedRows)
-//             if (!deletedRows.containsID ((int)r["id"]))
-//                retVal.addRow (r);
-//          return retVal;
-//       }
+         ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+                      
+         foreach(IBlueRow r in td)
+         {
+            truly.addRow(r);
+         }
 
-//       public virtual IBlueRowCollection getTrulyDeletedRows ()
-//       {
-//          BlueTable retVal = new BlueTable (schema);
-//          foreach (IBlueRow r in deletedRows)
-//             if (!addedRows.containsID ((int)r["id"]))
-//                retVal.addRow (r);
-//          return retVal;
-//       }
+         return truly;      
+      }
+      
+      public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
+      {
+       
+         BlueTable truly = new BlueTable(addedRows.schema);
+         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-//       public virtual IBlueRowCollection getTrulyChangedRows ()
-//       {
-//          BlueTable retVal = new BlueTable (schema);
-//          foreach (IBlueRow r in addedRows)
-//             if (deletedRows.containsID ((int)r["id"]))
-//                retVal.addRow (r);
-//          return retVal;
-//       }
+         CollectionHelpers.EqualDelegate ed = 
+            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
+         ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+         ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
+         
+         foreach(IBlueRow r in tc)
+         {
+            truly.addRow(r);
+         }   
+        
+         return truly;      
+      }
+      
+      public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
+      {        
+         BlueTable truly = new BlueTable(deletedRows.schema);
+         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-//       public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
-//       {        
-//          BlueTable truly = new BlueTable(deletedRows.schema);
-//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+         CollectionHelpers.EqualDelegate ed = 
+            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-//          CollectionHelpers.EqualDelegate ed = 
-//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+         ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+         ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
+         
+         foreach(IBlueRow r in tco)
+         {
+            truly.addRow(r);
+         }         
 
-//          ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+         return truly;         
+      }
+      
+      public TablePacket toTablePacket ()
+      {
+         return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
+                                 BlueDataUtility.toDataTable (deletedRows));
+                                 
+      }
 
-//          foreach(IBlueRow r in td)
-//          {
-//             truly.addRow(r);
-//          }
+      public IBlueTableSchema schema 
+      {
+         get
+         {
+            IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
+               addedRows.schema != null ? addedRows.schema : deletedRows.schema;
+            return retVal;
+         }
+      }
 
-//          return truly;      
-//       }
+      /*          
+       status  D total  A total  D this   A This   D' total A' Total 
+         x  0  0  0  0  0  0  
+            0  0  0  1     1 this+  
+            0  0  1  0  1this    
+            0  0  1  1  1 this   1 this+  
+         x  0  1  0  0        
+         error 0  1  0  1        
+            0  1  1  0     1 total- 
+            0  1  1  1     1 total-, 1 this +   
+         x  1  0  0  0        
+            1  0  0  1     1 this+  
+         error 1  0  1  0        
+         error 1  0  1  1        
+         x  1  1  0  0        
+         error 1  1  0  1        
+            1  1  1  0     1 total -1  keep the old delete and remove the old add
+            1  1  1  1     1 total-, 1 this +   
+      
+         d' = dthis - (dthis intersect a) + d // keep the old delete
+         a' = a - (dthis intersect a) + athis // keep the new add       
+       * */
+      public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
+      {       
+         if ( packets.Count == 0 )
+         {
+            return new DataPacket(
+               new BlueTable(new DataTable()),
+               new BlueTable(new DataTable()),
+               new BlueTable(new DataTable()));                    
+         }
 
-//       public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
-//       {
+         DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
+         
+         CollectionHelpers.EqualDelegate ed = 
+            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-//          BlueTable truly = new BlueTable(addedRows.schema);
-//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+         ArrayList added = new ArrayList();
+         ArrayList deleted = new ArrayList();
+         BlueTable dt = null;
+         BlueTable at = null;
+         IBlueRowCollection nt = null;
 
-//          CollectionHelpers.EqualDelegate ed = 
-//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+         foreach(DataPacket dp in packets)
+         {               
+            ArrayList oldDeleted = deleted;
+            ArrayList oldAdded = added;
 
-//          ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
-//          ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
+            added = new ArrayList();
+            deleted = new ArrayList();
+            if ( dt == null )
+            {
+               dt = new BlueTable(dp.schema);
+               at = new BlueTable(dp.schema);
+            }
+            
+            deleted.AddRange(oldDeleted);
+            deleted.AddRange( CollectionHelpers.difference(
+               CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
+               oldDeleted, ed) );
 
-//          foreach(IBlueRow r in tc)
-//          {
-//             truly.addRow(r);
-//          }   
+            added.AddRange(dp.addedRows);
+            added.AddRange( CollectionHelpers.difference(
+               CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
+               added, ed) );
+            nt = dp.newRows;
+         }        
+         
+         foreach ( IBlueRow dr in added )
+         {
+            at.addRow(dr);
+         }
 
-//          return truly;      
-//       }
+         foreach ( IBlueRow dr in deleted )
+         {
+            dt.addRow(dr);
+         }
 
-//       public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
-//       {        
-//          BlueTable truly = new BlueTable(deletedRows.schema);
-//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+         return new DataPacket(nt,at,dt);            
+      }
 
-//          CollectionHelpers.EqualDelegate ed = 
-//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+      public DataPacket deepCopy ()
+      {
+         return new DataPacket (new OrderedBlueTable (newRows),
+            new OrderedBlueTable (newRows),
+            new BlueTable (deletedRows));
+      }
 
-//          ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
-//          ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
+      public virtual DataPacket createImmutable ()
+      {
+         return 
+         new DataPacket(
+               newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
+               addedRows, 
+               deletedRows);
+      }
+      #region ICloneable Members
 
-//          foreach(IBlueRow r in tco)
-//          {
-//             truly.addRow(r);
-//          }         
+      public object Clone()
+      {
+         return MemberwiseClone();
+      }
 
-//          return truly;         
-//       }
+      #endregion
 
-//       public TablePacket toTablePacket ()
-//       {
-//          return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
-//                                  BlueDataUtility.toDataTable (deletedRows));
+      public string toIncrementalIDString()
+      {
+         System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
+         sb.Append ("\ndeleted : ");
+         foreach (IBlueRow r in deletedRows)
+         {
+            sb.Append(r["id"]);
+            sb.Append(',');
+         }
+         sb.Append("\nadded : ");
+         foreach (IBlueRow r in addedRows)
+         {
+            sb.Append(r["id"]);
+            sb.Append(',');
+         }
+         sb.Append("\n");
+         return sb.ToString();
+      }
 
-//       }
-
-//       public IBlueTableSchema schema 
-//       {
-//          get
-//          {
-//             IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
-//                addedRows.schema != null ? addedRows.schema : deletedRows.schema;
-//             return retVal;
-//          }
-//       }
-
-//       /*          
-//        status  D total  A total  D this   A This   D' total A' Total 
-//          x  0  0  0  0  0  0  
-//             0  0  0  1     1 this+  
-//             0  0  1  0  1this    
-//             0  0  1  1  1 this   1 this+  
-//          x  0  1  0  0        
-//          error 0  1  0  1        
-//             0  1  1  0     1 total- 
-//             0  1  1  1     1 total-, 1 this +   
-//          x  1  0  0  0        
-//             1  0  0  1     1 this+  
-//          error 1  0  1  0        
-//          error 1  0  1  1        
-//          x  1  1  0  0        
-//          error 1  1  0  1        
-//             1  1  1  0     1 total -1  keep the old delete and remove the old add
-//             1  1  1  1     1 total-, 1 this +   
-
-//          d' = dthis - (dthis intersect a) + d // keep the old delete
-//          a' = a - (dthis intersect a) + athis // keep the new add       
-//        * */
-//       public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
-//       {       
-//          if ( packets.Count == 0 )
-//          {
-//             return new DataPacket(
-//                new BlueTable(new DataTable()),
-//                new BlueTable(new DataTable()),
-//                new BlueTable(new DataTable()));                    
-//          }
-
-//          DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
-
-//          CollectionHelpers.EqualDelegate ed = 
-//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
-
-//          ArrayList added = new ArrayList();
-//          ArrayList deleted = new ArrayList();
-//          BlueTable dt = null;
-//          BlueTable at = null;
-//          IBlueRowCollection nt = null;
-
-//          foreach(DataPacket dp in packets)
-//          {               
-//             ArrayList oldDeleted = deleted;
-//             ArrayList oldAdded = added;
-
-//             added = new ArrayList();
-//             deleted = new ArrayList();
-//             if ( dt == null )
-//             {
-//                dt = new BlueTable(dp.schema);
-//                at = new BlueTable(dp.schema);
-//             }
-
-//             deleted.AddRange(oldDeleted);
-//             deleted.AddRange( CollectionHelpers.difference(
-//                CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
-//                oldDeleted, ed) );
-
-//             added.AddRange(dp.addedRows);
-//             added.AddRange( CollectionHelpers.difference(
-//                CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
-//                added, ed) );
-//             nt = dp.newRows;
-//          }        
-
-//          foreach ( IBlueRow dr in added )
-//          {
-//             at.addRow(dr);
-//          }
-
-//          foreach ( IBlueRow dr in deleted )
-//          {
-//             dt.addRow(dr);
-//          }
-
-//          return new DataPacket(nt,at,dt);            
-//       }
-
-//       public DataPacket deepCopy ()
-//       {
-//          return new DataPacket (new OrderedBlueTable (newRows),
-//             new OrderedBlueTable (newRows),
-//             new BlueTable (deletedRows));
-//       }
-
-//       public virtual DataPacket createImmutable ()
-//       {
-//          return 
-//          new DataPacket(
-//                newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
-//                addedRows, 
-//                deletedRows);
-//       }
-//       #region ICloneable Members
-
-//       public object Clone()
-//       {
-//          return MemberwiseClone();
-//       }
-
-//       #endregion
-
-//       public string toIncrementalIDString()
-//       {
-//          System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
-//          sb.Append ("\ndeleted : ");
-//          foreach (IBlueRow r in deletedRows)
-//          {
-//             sb.Append(r["id"]);
-//             sb.Append(',');
-//          }
-//          sb.Append("\nadded : ");
-//          foreach (IBlueRow r in addedRows)
-//          {
-//             sb.Append(r["id"]);
-//             sb.Append(',');
-//          }
-//          sb.Append("\n");
-//          return sb.ToString();
-//       }
-
-//       public override string ToString()
-//       {
-//          return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
-//       }
-//    }
-// }
+      public override string ToString()
+      {
+         return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -1,5 +1,4 @@
-namespace BCG
-{
+namespace BCG {
    using Bts.DataStream;
    using bts;
 
@@ -10,28 +9,26 @@ namespace BCG
    using System.Reflection;
    using System.IO;
 
-   public class PrimitiveManager
-   {
+   public class PrimitiveManager {
       private IDictionary myTypeLookupTable = new Hashtable();
       private static PrimitiveManager mySelf;
       private ArrayList myAssemblyNames = new ArrayList();
 
-      private PrimitiveManager()
-      {
+      private PrimitiveManager() {
          myLoadAllTypes();
       }
 
-      private void myLoadAllTypes()
-      {
+      private void myLoadAllTypes() {
          AppDomain currentDomain = AppDomain.CurrentDomain;
          currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
          loadAssembly(this.GetType().Assembly.FullName);
-
-         foreach (string assemblyName in Configuration.get<DataStreamConfig>().Assemblies)
-         {
+         var assemblies = new string[] { "datastream.dll", "sharedstreams.dll" };
+         var config_assemblies = Configuration.get<DataStreamConfig>().Assemblies;
+         bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}");
+         foreach (string assemblyName in assemblies/*config_assemblies*/) {
+            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
             string resolvedAssemblyPath = assemblyName;
-            if (!System.IO.Path.IsPathRooted(assemblyName))
-            {
+            if (!System.IO.Path.IsPathRooted(assemblyName)) {
                string appPath = Path.GetDirectoryName(
                               Assembly.GetAssembly(typeof(PrimitiveManager)).CodeBase.Substring("file:///".Length));
                resolvedAssemblyPath = System.IO.Path.GetFullPath(System.IO.Path.Combine(appPath, assemblyName));
@@ -40,8 +37,7 @@ namespace BCG
          }
       }
 
-      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
-      {
+      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
          foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
             if (args.Name.Equals(a.FullName))
                return a;
@@ -52,8 +48,8 @@ namespace BCG
       /// Loads all the Stream types associated with the assembly
       /// </summary>
       /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName)
-      {
+      public void loadAssembly(string assemblyName) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
             return;
          myAssemblyNames.Add(assemblyName);
@@ -65,13 +61,11 @@ namespace BCG
             if (!type.IsInterface
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
-               && type.IsPublic)
-            {
+               && type.IsPublic) {
                bool isVisible = true;
                string userRep = createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
-                  if (att is DataStreamUserRespresentation)
-                  {
+                  if (att is DataStreamUserRespresentation) {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
                      isVisible = dsatt.isVisible;
                      userRep = isVisible ?
@@ -79,16 +73,14 @@ namespace BCG
                         : null;
                      break;
                   }
-               if (isVisible)
-               {
+               if (isVisible) {
                   Log.assert(myTypeLookupTable[userRep] == null);
                   myTypeLookupTable[userRep] = type;
                }
             }
       }
 
-      public static string createDefaultName(string typeName)
-      {
+      public static string createDefaultName(string typeName) {
          if (!typeName.EndsWith("Stream"))
             return typeName;
          string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
@@ -100,8 +92,7 @@ namespace BCG
 
 
 
-      public static PrimitiveManager instance
-      {
+      public static PrimitiveManager instance {
          get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
       }
 
@@ -110,8 +101,7 @@ namespace BCG
       /// </summary>
       /// <param name="friendlyName"></param>
       /// <returns></returns>
-      public string find(string friendlyName)
-      {
+      public string find(string friendlyName) {
          if (myTypeLookupTable[friendlyName] == null)
             throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          return (myTypeLookupTable[friendlyName] as Type).Name;
@@ -124,11 +114,9 @@ namespace BCG
       /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
       /// is the desired one</param>
       /// <returns></returns>
-      public ViewStream build(string friendlyName, ICollection args)
-      {
+      public ViewStream build(string friendlyName, ICollection args) {
          object[] constructorArgs = null;
-         if (args != null && args.Count > 0)
-         {
+         if (args != null && args.Count > 0) {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
@@ -144,8 +132,7 @@ namespace BCG
       /// Returns the name of all the primitives 
       /// </summary>
       /// <returns></returns>
-      public string[] getPrimitiveNames()
-      {
+      public string[] getPrimitiveNames() {
          string[] retVal = new string[myTypeLookupTable.Count];
          myTypeLookupTable.Keys.CopyTo(retVal, 0);
          return retVal;
@@ -155,20 +142,17 @@ namespace BCG
       /// Return the types of all the streams
       /// </summary>
       /// <returns></returns>
-      public Type[] getStreamTypes()
-      {
+      public Type[] getStreamTypes() {
          Type[] retVal = new Type[myTypeLookupTable.Count];
          myTypeLookupTable.Values.CopyTo(retVal, 0);
          return retVal;
       }
 
-      public ICollection getConstructorSigns(string primitiveName)
-      {
+      public ICollection getConstructorSigns(string primitiveName) {
          ArrayList retVal = new ArrayList();
          Log.assert(myTypeLookupTable[primitiveName] != null);
          Type t = myTypeLookupTable[primitiveName] as Type;
-         foreach (ConstructorInfo ci in t.GetConstructors())
-         {
+         foreach (ConstructorInfo ci in t.GetConstructors()) {
             string sign = "";
             sign = t.Name + " (";
             foreach (ParameterInfo pi in ci.GetParameters())
@@ -181,8 +165,7 @@ namespace BCG
          return retVal;
       }
 
-      public string getAssemblyName(string primitiveName)
-      {
+      public string getAssemblyName(string primitiveName) {
          Log.assert(myTypeLookupTable[primitiveName] != null);
          return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
       }

src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs

@@ -42,7 +42,7 @@ namespace BCG
                Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
             if (myStateTable != null)
                value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
-         }
+      }
          remove
          {
             // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
@@ -64,6 +64,7 @@ namespace BCG
 
       public void update(Stream[] streams)
       {
+         bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
          //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
          //data element for that tag in the ViewStreamDatabase
          if (myOnUpdateDelegate != null)

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs

@@ -3,16 +3,15 @@ using System.IO;
 using System.Collections;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
    /// </summary>
-   public class ClientDeserializationController : IClientDeserializer
-   {
+   public class ClientDeserializationController : IClientDeserializer {
       private IClientDeserializer myDeserializer;
-		
-      public ClientDeserializationController () {}
+      private int _n_desers = 0;
+
+      public ClientDeserializationController() { }
 
       #region IClientDeserializer Members
 
@@ -23,42 +22,41 @@ namespace BCG
       /// <param name="streams"></param>
       /// <param name="theOps"></param>
       /// <param name="theRows"></param>
-      public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         if (myDeserializer != null)
-         {
-            myDeserializer.deserialize (streams, out theOps, out theRows);
-         }
-         else 
-         {
-            Stream totalStream = UtilityMethods.getAggregatedStream (streams);
+      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
+         bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
+         if (myDeserializer != null) {
+            myDeserializer.deserialize(streams, out theOps, out theRows);
+         } else {
+            Stream totalStream = UtilityMethods.getAggregatedStream(streams);
             long initialPos = totalStream.Position;
-            BinaryReader br = new BinaryReader (totalStream);
+            BinaryReader br = new BinaryReader(totalStream);
             byte op;
-            try 
-            {
-               op = br.ReadByte ();
-            }
-            catch (EndOfStreamException)
-            {
+            try {
+               op = br.ReadByte();
+            } catch (EndOfStreamException) {
                op = Byte.MaxValue;
             }
-            if (op == StreamOpCode.create)
-            {
-               br.ReadString (); //pubsubclientdeserializer
-               string deserName = br.ReadString ();   
-               myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
+            if (op == StreamOpCode.create) {
+               br.ReadString(); //pubsubclientdeserializer
+               string deserName = br.ReadString();
+               myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
                   as IClientDeserializer;
                totalStream.Position = initialPos;
-               myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
-            }
-            else
-            {
+               myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
+            } else {
                // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
-               theOps = new byte[]{};
-               theRows = new object[]{};
+               theOps = new byte[] { };
+               theRows = new object[] { };
             }
          }
+         var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
+         int i = 0;
+         foreach (var m in msgs) {
+            bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
+         }
+         _n_desers++;
+         // log it here
+
       }
 
 

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs

@@ -5,61 +5,61 @@ using System.IO;
 using BCG.Utilities;
 //using BCG.Utilities.BlueConfiguration;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
    /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) 
    /// </summary>
-   internal class ClientUniversalRowDeserializer : IClientRowDeserializer
-   {
+   internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
       private IBlueTableSchema mySchema;
       private int myNullMapLength = 0;
       private PrimitiveDeserializationHelper myDeserializationHelper;
       private BitArray myNullMap;
       private int[] mySerIndexToSchemaIndexMap;
-		
-      public ClientUniversalRowDeserializer (IBlueTableSchema schema)
-      {
-         myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
-         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
-         myInitReorderSchema ();
 
-         
+      public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
+         myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
+         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
+         myInitReorderSchema();
+
+
       }
 
       #region Schema Reordering initialization
-      private void myInitReorderSchema ()
-      {
-         int idColIndex = mySchema["id"].colNo;
-         int[] temp = new int [mySchema.columnInfo.Length];
-         foreach (ColumnInfo ci in mySchema.columnInfo)
-         {
-            temp[ci.colNo] =
-               ci.colNo == idColIndex ? 0 :
-               ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
+      private void myInitReorderSchema() {
+         try {
+            int idColIndex = mySchema["id"].colNo;
+            int[] temp = new int[mySchema.columnInfo.Length];
+            foreach (ColumnInfo ci in mySchema.columnInfo) {
+               temp[ci.colNo] =
+                  ci.colNo == idColIndex ? 0 :
+                  ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
+            }
+            mySerIndexToSchemaIndexMap = new int[temp.Length];
+            for (int i = 0; i < temp.Length; i++)
+               mySerIndexToSchemaIndexMap[temp[i]] = i;
+         } catch (Exception ex) {
+            string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
+            string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
+            bts.Log.Gen.error(err_msg);
+            throw new ApplicationException(err_msg);
          }
-         mySerIndexToSchemaIndexMap = new int[temp.Length];
-         for (int i = 0; i < temp.Length; i ++)
-            mySerIndexToSchemaIndexMap [temp[i]] = i;
       }
       #endregion
 
       #region IClientRowDeserializer Members
 
-      public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
-      {
-         br.ReadInt32 ();  //length information for the row not needed
-         myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
-         object[] values = new object [mySchema.columnNames.Length];
-         for (int i = 0; i < values.Length; i++)
-         {
+      public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
+         br.ReadInt32();  //length information for the row not needed
+         myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
+         object[] values = new object[mySchema.columnNames.Length];
+         for (int i = 0; i < values.Length; i++) {
             int colNo = mySerIndexToSchemaIndexMap[i];
-            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
+            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
          }
 
 
-             r = new BlueRow(mySchema, values);
+         r = new BlueRow(mySchema, values);
 
       }
 

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs

@@ -1,22 +1,149 @@
 using System;
-using System.Collections;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+
+namespace BCG {
+   public class StreamIOMessage {
+      public enum StreamOp {
+         Create = 0,
+         Add = 1,             //adds a single row
+         Remove = 2,          //removes a single row
+         Change = 3,          //changes a single row
+         MultipleRemoveAdd = 4,//removes and adds multiple rows
+         MultipleChange = 5 //changes multiples rows          
+      }
+      public class CreateMessage {
+         public IBlueRowCollection rows;
+         public override string ToString() {
+            string rv = "Create message ";
+            if (rows == null) {
+               rv += " rows: null";
+               return rv;
+            }
+            foreach (IBlueRow r in rows) {
+               rv += $"id: {r["id"]} ";
+            }
+            return rv;
+         }
+      }
+      public class AddMessage {
+         public IBlueRow added_row;
+      }
+      public class RemoveMessage {
+         public IBlueRow removed_row;
+      }
+      public class ChangeMessage {
+         public IBlueRow changed_row;
+      }
+      public class MultipleRemoveAddMessage {
+         public IBlueRowCollection removed;
+         public IBlueRowCollection added;
+         public override string ToString() {
+            string rv = "MultipleRemoveAdd removed: ";
+            foreach (IBlueRow r in removed) {
+               rv += $"id: {r["id"]} ";
+            }
+            rv += "added: ";
+            foreach (IBlueRow r in added) {
+               rv += $"id: {r["id"]} ";
+            }
+            return rv;
+         }
+      }
+      public class MultipleChangeMessage {
+         public IBlueRowCollection changed;
+      }
+
+      public StreamOp op;
+      public CreateMessage create_message = null;
+      public AddMessage add_message = null;
+      public RemoveMessage remove_message = null;
+      public ChangeMessage change_message = null;
+      public MultipleRemoveAddMessage multiple_remove_add_message = null;
+      public MultipleChangeMessage multiple_change_message = null;
+      public StreamIOMessage(byte op_, object data) {
+         op = (StreamIOMessage.StreamOp)(op_);
+         switch (op) {
+            case StreamIOMessage.StreamOp.Create:
+               IBlueRowCollection state = null;
+               // TODO, not sure why it can be an array and by itself
+               if (data is IBlueRowCollection) state = data as IBlueRowCollection;
+               else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
+               create_message = new CreateMessage() { rows = state };
+               break;
+            case StreamIOMessage.StreamOp.Add:
+               bts.Log.assert(false);
+               break;
+            case StreamIOMessage.StreamOp.Remove:
+               bts.Log.assert(false);
+               break;
+            case StreamIOMessage.StreamOp.Change:
+               bts.Log.assert(false);
+               break;
+            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
+               IBlueRowCollection[] row_args = data as IBlueRowCollection[];
+               multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
+               break;
+            case StreamIOMessage.StreamOp.MultipleChange:
+               //IBlueRowCollection[] row_args = data as IBlueRowCollection[];
+               multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
+               break;
+         }
+      }
+
+
+      public override string ToString() {
+         switch (op) {
+            case StreamIOMessage.StreamOp.Create:
+               return create_message.ToString();
+               break;
+            case StreamIOMessage.StreamOp.Add:
+               bts.Log.assert(false);
+               return String.Empty;
+               break;
+            case StreamIOMessage.StreamOp.Remove:
+               bts.Log.assert(false);
+               return String.Empty;
+               break;
+            case StreamIOMessage.StreamOp.Change:
+               bts.Log.assert(false);
+               return String.Empty;
+               break;
+            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
+               return multiple_remove_add_message.ToString();
+               break;
+            case StreamIOMessage.StreamOp.MultipleChange:
+               return multiple_change_message.ToString();
+               break;
+         }
+         bts.Log.assert(false);
+         return String.Empty;
+      }
+      public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
+         int i = 0;
+         List<StreamIOMessage> rv = new List<StreamIOMessage>();
+         foreach (byte op in ops) {
+            object data = op_data.GetValue(i++);
+            rv.Add(new StreamIOMessage(op, data));
+         }
+         return rv;
+      }
+   }
 
-namespace BCG
-{
    /// <summary>
    /// Performs deserialization inside the client app. The client gets either the complete data set or the
    /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
    /// </summary>
-   public interface IClientDeserializer 
-   {
+   public interface IClientDeserializer {
       /// <summary>
       /// deserializes the data received by the client from the pubsub
       /// </summary>
       /// <param name="serializedData">the data in byte format received from pubsub</param>
       /// <param name="theOps">the operations that the data maps onto</param>
       /// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
-      void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
+      void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
    }
-
 }
+
+

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs

@@ -2,13 +2,11 @@ using System;
 using System.IO;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. 
    /// </summary>
-   public interface IClientRowDeserializer
-   {
-      void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
+   public interface IClientRowDeserializer {
+      void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs

@@ -16,7 +16,7 @@ namespace BCG {
 
       public PubTablePublisher(string tag, PublisherCore publisherCore) {
          _tag = tag;
-         bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}");
+         bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}");
          myPubCore = publisherCore;
          mySerializer = new BlueDataSerializationController();
       }
@@ -54,10 +54,14 @@ namespace BCG {
                                  out data);
          bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
          myPublish(data, rows.schema.tableName);
+         // byte[] ops;
+         // Array rows_ds;
+         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
       }
 
+      //private ClientDeserializationController _test_deserializer = new ClientDeserializationController();
       public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
-         byte[] data;
+         byte[] data = null;
          if (myIsRealChange(delRows, addRows))
             batchChange(addRows);
          else {
@@ -66,6 +70,9 @@ namespace BCG {
                                     out data);
             myPublish(data, delRows.schema.tableName);
          }
+         // byte[] ops;
+         // Array rows_ds;
+         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
       }
 
       public virtual void batchChange(IBlueRowCollection rows) {

src/DataStreamSuite/DataStreamCore/Misc/IPubSubRouter.cs

@@ -1,20 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace BCG
-{
-   public delegate void PubSubTagURLChangeDelegate(List<string> tagsMoving, string newURL);
-   
-   public interface IPubSubRouter
-   {
-
-      event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
-      event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
-      List<string> getAllPubSubSubURLs();
-      List<string> getAllPubSubPubURLs();
-      string getSubURLForTag(string tag);
-      string getPubURLForTag(string tag);
-   }
-}

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y

@@ -23,6 +23,7 @@
 
 %token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, CS_STRING_CONSTANT, NULL_CONSTANT
 %token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
+%token LINE_COMMENT
 
 %nonassoc EQUALS
 
@@ -49,6 +50,12 @@ Statement :
    | IDENTIFIER EQUALS Statement {
       $$ = new Assignment($1.Identifier, $3);
    }
+   | LINE_COMMENT Statement {
+      $$ = $2;
+   }
+   | Statement LINE_COMMENT {
+      $$ = $1;
+   }
    ;
 
 Arguments :

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -26,6 +26,9 @@ Comma                    ,
 QuoteOpen                \[
 QuoteClose               \]
 
+/* Comments */
+LineComment \/\/[^\n\r]*(\n|\r|\r\n)
+
 %%
 
 {Equals} {
@@ -92,4 +95,8 @@ QuoteClose               \]
    return (int)DataStreamExpressionToken.IDENTIFIER;
 }
 
+{LineComment} {
+   return (int)DataStreamExpressionToken.LINE_COMMENT;
+}
+
 %%

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -48,14 +48,26 @@ public class PrimitiveExpression : StatementExpression {
       return PrimitiveManager.instance.build(myName, resolvedArgs);
    }
 
+   string myPrintSingleArg(int atColumn, object? o, IList<string> activeBindings) {
+      /// Unwrap <see cref="Argument"/> to get its underlying value
+      while (o is Argument arg) {
+         o = arg.Obj;
+         // Safeguard against infinite loop
+         if (ReferenceEquals(o, arg)) {
+            throw new ArgumentException(
+               $"Detected circular reference in argument " +
+               $"('{nameof(arg)}.{nameof(arg.Obj)}' is '{nameof(arg)}' itself)");
+         }
+      }
 
-   string myPrintSingleArg(int atColumn, object o, IList<string> activeBindings) {
+      string result = new string(' ', atColumn);
+      if (o is null) {
+         return result;
+      }
       if (o is StatementExpression)
          return new string(' ', atColumn) +
             ((o as StatementExpression)?.print(atColumn, activeBindings) ?? string.Empty);
       else {
-         string result = new string(' ', atColumn);
-
          if (o is string)
             result += makeCSString((string)o);
          else if (o is DateTime)

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -3,9 +3,10 @@
   <ItemGroup>
     <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\pubsub\pubsub.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\PubSub\PubSub.csproj" />
   </ItemGroup>
 
   <PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -1,35 +1,84 @@
 using System;
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
+using Bts.DataStreamExpressions;
+using System.Collections.Generic;
 
 using BCG;
 
-bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
-mbts.config.Configuration.initialize();
 
 bts.Log.Gen.info($"Starting DSPublisher");
 Console.WriteLine("Starting DSPublisher...");
-// SimpleTCPClient.start("127.0.0.1", 12345);
-// Task.Delay(1000000000).Wait();
-
-//TestPubliser.publish();
-//TestPubliser.subscribe();
-// TestPublisher ts = new TestPublisher(false);
-// ts.keep_publishing(TimeSpan.FromSeconds(10));
-
-
-_subscribe_to_stream();
-
-void _subscribe_to_stream() {
-   BCG.ViewStream vs = new TableStream("literal");
-   vs.onChange += on_change;
+static void init(string[] args)
+{
+   Console.WriteLine("Hit key to continue");
+   Console.ReadLine();
+   Func<string, string, Task> login = async (l, p) =>
+   {
+      var login = l;
+      var passwd = p;
+      var service_ctx = bts.api.core.ServiceContext.create();
+      await service_ctx.login(login, passwd);
+   };
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   Task.Run(async () => await login("bts", String.Empty));
 }
 
-while (true) {
-   System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
-   ViewStreamDatabase.instance.doWork();
+init(args);
+_publish_streams();
+
+static ViewStream build_stream(string stream_expression)
+{
+   try
+   {
+      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+      return rv;
+
+   }
+   catch (Exception ex)
+   {
+      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+      return null;
+   }
 }
 
 
-void on_change(object obj) {
-   DataPacket dp = obj as DataPacket;
-   dp.newRows.print_table((col) => { return 10; });
+void _publish_streams()
+{
+   PubTableBroker pb_broker = new PubTableBroker();
+   CloudStreamManager csm = new CloudStreamManager();
+   // get it from config
+   Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
+        {"bbo","bbo()"},
+        {"instruments","instruments()"},
+        //{"timesales","timesales(10000)"},
+        {"books","books()"},
+      // {"seconds_1_10","second(1,10)"},
+        {"literal_420","literal(\"x\", \"420.0\")"},
+      //  {"bbo_ts","timeSeries(bbo())"}
+   };
+   foreach (var kv in streams_to_publish)
+   {
+      var name = kv.Key;
+      var exp = kv.Value;
+      BCG.ViewStream vs = build_stream(exp);
+      ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
+      vs.onChange += on_change;
+   }
+   while (true)
+   {
+      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
+      ViewStreamDatabase.instance.doWork();
+      csm.doWork();
+   }
+}
+
+
+
+void on_change(object obj)
+{
+   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -1,13 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-  <ItemGroup>
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\pubsub\pubsub.csproj" />
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <!-- <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> -->
-  </ItemGroup>
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net7.0-windows</TargetFramework>
@@ -20,11 +12,13 @@
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\BlueRegistry\BlueRegistry.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
     <ProjectReference Include="..\GuiUtilities\GuiUtilities.csproj" />
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStreamQueryAnalyzer/HScrollGrid.resx

@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="utf-8"?>
+<root>
+  <!-- 
+    Microsoft ResX Schema 
+    
+    Version 2.0
+    
+    The primary goals of this format is to allow a simple XML format 
+    that is mostly human readable. The generation and parsing of the 
+    various data types are done through the TypeConverter classes 
+    associated with the data types.
+    
+    Example:
+    
+    ... ado.net/XML headers & schema ...
+    <resheader name="resmimetype">text/microsoft-resx</resheader>
+    <resheader name="version">2.0</resheader>
+    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+        <value>[base64 mime encoded serialized .NET Framework object]</value>
+    </data>
+    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+        <comment>This is a comment</comment>
+    </data>
+                
+    There are any number of "resheader" rows that contain simple 
+    name/value pairs.
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
+    mimetype set.
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
+    extensible. For a given mimetype the value must be set accordingly:
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
+    read any of the formats listed below.
+    
+    mimetype: application/x-microsoft.net.object.binary.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+            : and then encoded with base64 encoding.
+    
+    mimetype: application/x-microsoft.net.object.soap.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+            : and then encoded with base64 encoding.
+
+    mimetype: application/x-microsoft.net.object.bytearray.base64
+    value   : The object must be serialized into a byte array 
+            : using a System.ComponentModel.TypeConverter
+            : and then encoded with base64 encoding.
+    -->
+  <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+    <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+    <xsd:element name="root" msdata:IsDataSet="true">
+      <xsd:complexType>
+        <xsd:choice maxOccurs="unbounded">
+          <xsd:element name="metadata">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" />
+              </xsd:sequence>
+              <xsd:attribute name="name" use="required" type="xsd:string" />
+              <xsd:attribute name="type" type="xsd:string" />
+              <xsd:attribute name="mimetype" type="xsd:string" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="assembly">
+            <xsd:complexType>
+              <xsd:attribute name="alias" type="xsd:string" />
+              <xsd:attribute name="name" type="xsd:string" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="data">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+                <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+              <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+              <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="resheader">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" />
+            </xsd:complexType>
+          </xsd:element>
+        </xsd:choice>
+      </xsd:complexType>
+    </xsd:element>
+  </xsd:schema>
+  <resheader name="resmimetype">
+    <value>text/microsoft-resx</value>
+  </resheader>
+  <resheader name="version">
+    <value>2.0</value>
+  </resheader>
+  <resheader name="reader">
+    <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+  <resheader name="writer">
+    <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+</root>

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.Designer.cs

@@ -5,16 +5,16 @@
       /// <summary>
       /// Clean up any resources being used.
       /// </summary>
-      protected override void Dispose (bool disposing)
+      protected override void Dispose(bool disposing)
       {
          if (disposing)
          {
             if (components != null)
             {
-               components.Dispose ();
+               components.Dispose();
             }
          }
-         base.Dispose (disposing);
+         base.Dispose(disposing);
       }
 
       #region Windows Form Designer generated code
@@ -22,500 +22,549 @@
       /// Required method for Designer support - do not modify
       /// the contents of this method with the code editor.
       /// </summary>
-      private void InitializeComponent ()
+      private void InitializeComponent()
       {
-         this.components = new System.ComponentModel.Container();
+         components = new System.ComponentModel.Container();
+         System.Windows.Forms.DataGridViewCellStyle dataGridViewCellStyle1 = new System.Windows.Forms.DataGridViewCellStyle();
+         System.Windows.Forms.DataGridViewCellStyle dataGridViewCellStyle2 = new System.Windows.Forms.DataGridViewCellStyle();
          System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(MainForm));
-         this.myQueryTextBox = new System.Windows.Forms.RichTextBox();
-         this.myTabControl = new System.Windows.Forms.TabControl();
-         this.myViewTabPage = new System.Windows.Forms.TabPage();
-         this.myDataGrid = new BCG.HScrollGrid();
-         this.myCSharpTabPage = new System.Windows.Forms.TabPage();
-         this.myCSharpText = new System.Windows.Forms.RichTextBox();
-         this.myErrorTabPage = new System.Windows.Forms.TabPage();
-         this.myErrorTextBox = new System.Windows.Forms.RichTextBox();
-         this.myTableListTabPage = new System.Windows.Forms.TabPage();
-         this.myTableListBox = new BCG.Utilities.XPControls.ListBoxXP();
-         this.myTableListTopPanel = new System.Windows.Forms.Panel();
-         this.myTableFilterBox = new System.Windows.Forms.TextBox();
-         this.myTableFilterLabel = new System.Windows.Forms.Label();
-         this.myPrimitivesPage = new System.Windows.Forms.TabPage();
-         this.myTreeView = new System.Windows.Forms.TreeView();
-         this.myConsoleTabPage = new System.Windows.Forms.TabPage();
-         this.myConsoleRichTextBox = new System.Windows.Forms.RichTextBox();
-         this.myTimer = new System.Windows.Forms.Timer(this.components);
-         this.mySplitter = new System.Windows.Forms.Splitter();
-         this.myStatusStrip = new System.Windows.Forms.StatusStrip();
-         this.myToolStripStatusLabelTime = new System.Windows.Forms.ToolStripStatusLabel();
-         this.myToolStripStatusLabelRows = new System.Windows.Forms.ToolStripStatusLabel();
-         this.myMenuStrip = new System.Windows.Forms.MenuStrip();
-         this.myToolStripMenuItemFile = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemOpen = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemSave = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemSaveAs = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemExit = new System.Windows.Forms.ToolStripMenuItem();
-         this.testGridToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemQuery = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemExecute = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemExecSel = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemFreeze = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemStep = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemTools = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemLoadAss = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemTest = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemToolsSep = new System.Windows.Forms.ToolStripSeparator();
-         this.myToolStripMenuItemClearConsole = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemSite = new System.Windows.Forms.ToolStripMenuItem();
-         this.libraryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
-         this.myTabControl.SuspendLayout();
-         this.myViewTabPage.SuspendLayout();
-         ((System.ComponentModel.ISupportInitialize)(this.myDataGrid)).BeginInit();
-         this.myCSharpTabPage.SuspendLayout();
-         this.myErrorTabPage.SuspendLayout();
-         this.myTableListTabPage.SuspendLayout();
-         this.myTableListTopPanel.SuspendLayout();
-         this.myPrimitivesPage.SuspendLayout();
-         this.myConsoleTabPage.SuspendLayout();
-         this.myStatusStrip.SuspendLayout();
-         this.myMenuStrip.SuspendLayout();
-         this.SuspendLayout();
+         myQueryTextBox = new System.Windows.Forms.RichTextBox();
+         myTabControl = new System.Windows.Forms.TabControl();
+         myViewTabPage = new System.Windows.Forms.TabPage();
+         myDataGrid = new HScrollGrid();
+         myCSharpTabPage = new System.Windows.Forms.TabPage();
+         myCSharpText = new System.Windows.Forms.RichTextBox();
+         myErrorTabPage = new System.Windows.Forms.TabPage();
+         myErrorTextBox = new System.Windows.Forms.RichTextBox();
+         myTableListTabPage = new System.Windows.Forms.TabPage();
+         myTableListBox = new Utilities.XPControls.ListBoxXP();
+         myTableListTopPanel = new System.Windows.Forms.Panel();
+         myTableFilterBox = new System.Windows.Forms.TextBox();
+         myTableFilterLabel = new System.Windows.Forms.Label();
+         myPrimitivesPage = new System.Windows.Forms.TabPage();
+         myTreeView = new System.Windows.Forms.TreeView();
+         myConsoleTabPage = new System.Windows.Forms.TabPage();
+         myConsoleRichTextBox = new System.Windows.Forms.RichTextBox();
+         myTimer = new System.Windows.Forms.Timer(components);
+         mySplitter = new System.Windows.Forms.Splitter();
+         myStatusStrip = new System.Windows.Forms.StatusStrip();
+         myToolStripStatusLabelTime = new System.Windows.Forms.ToolStripStatusLabel();
+         myToolStripStatusLabelRows = new System.Windows.Forms.ToolStripStatusLabel();
+         toolStripStatusLabel1 = new System.Windows.Forms.ToolStripStatusLabel();
+         myMenuStrip = new System.Windows.Forms.MenuStrip();
+         myToolStripMenuItemFile = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemOpen = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemSave = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemSaveAs = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemExit = new System.Windows.Forms.ToolStripMenuItem();
+         testGridToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemQuery = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemExecute = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemExecSel = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemFreeze = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemStep = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemTools = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemLoadAss = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemTest = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemToolsSep = new System.Windows.Forms.ToolStripSeparator();
+         myToolStripMenuItemClearConsole = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemSite = new System.Windows.Forms.ToolStripMenuItem();
+         libraryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+         myTabControl.SuspendLayout();
+         myViewTabPage.SuspendLayout();
+         ((System.ComponentModel.ISupportInitialize)myDataGrid).BeginInit();
+         myCSharpTabPage.SuspendLayout();
+         myErrorTabPage.SuspendLayout();
+         myTableListTabPage.SuspendLayout();
+         myTableListTopPanel.SuspendLayout();
+         myPrimitivesPage.SuspendLayout();
+         myConsoleTabPage.SuspendLayout();
+         myStatusStrip.SuspendLayout();
+         myMenuStrip.SuspendLayout();
+         SuspendLayout();
          // 
          // myQueryTextBox
          // 
-         this.myQueryTextBox.AutoWordSelection = true;
-         this.myQueryTextBox.BackColor = System.Drawing.Color.White;
-         this.myQueryTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myQueryTextBox.Dock = System.Windows.Forms.DockStyle.Top;
-         this.myQueryTextBox.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         this.myQueryTextBox.ForeColor = System.Drawing.Color.Black;
-         this.myQueryTextBox.Location = new System.Drawing.Point(0, 24);
-         this.myQueryTextBox.Name = "myQueryTextBox";
-         this.myQueryTextBox.Size = new System.Drawing.Size(904, 208);
-         this.myQueryTextBox.TabIndex = 0;
-         this.myQueryTextBox.Text = "";
+         myQueryTextBox.AutoWordSelection = true;
+         myQueryTextBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myQueryTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myQueryTextBox.Dock = System.Windows.Forms.DockStyle.Top;
+         myQueryTextBox.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         myQueryTextBox.ForeColor = System.Drawing.Color.White;
+         myQueryTextBox.Location = new System.Drawing.Point(0, 24);
+         myQueryTextBox.Name = "myQueryTextBox";
+         myQueryTextBox.Size = new System.Drawing.Size(904, 208);
+         myQueryTextBox.TabIndex = 0;
+         myQueryTextBox.Text = "";
          // 
          // myTabControl
          // 
-         this.myTabControl.Appearance = System.Windows.Forms.TabAppearance.FlatButtons;
-         this.myTabControl.Controls.Add(this.myViewTabPage);
-         this.myTabControl.Controls.Add(this.myCSharpTabPage);
-         this.myTabControl.Controls.Add(this.myErrorTabPage);
-         this.myTabControl.Controls.Add(this.myTableListTabPage);
-         this.myTabControl.Controls.Add(this.myPrimitivesPage);
-         this.myTabControl.Controls.Add(this.myConsoleTabPage);
-         this.myTabControl.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myTabControl.Location = new System.Drawing.Point(0, 232);
-         this.myTabControl.Multiline = true;
-         this.myTabControl.Name = "myTabControl";
-         this.myTabControl.SelectedIndex = 0;
-         this.myTabControl.Size = new System.Drawing.Size(904, 335);
-         this.myTabControl.TabIndex = 1;
+         myTabControl.Appearance = System.Windows.Forms.TabAppearance.FlatButtons;
+         myTabControl.Controls.Add(myViewTabPage);
+         myTabControl.Controls.Add(myCSharpTabPage);
+         myTabControl.Controls.Add(myErrorTabPage);
+         myTabControl.Controls.Add(myTableListTabPage);
+         myTabControl.Controls.Add(myPrimitivesPage);
+         myTabControl.Controls.Add(myConsoleTabPage);
+         myTabControl.Dock = System.Windows.Forms.DockStyle.Fill;
+         myTabControl.Location = new System.Drawing.Point(0, 232);
+         myTabControl.Multiline = true;
+         myTabControl.Name = "myTabControl";
+         myTabControl.SelectedIndex = 0;
+         myTabControl.Size = new System.Drawing.Size(904, 335);
+         myTabControl.TabIndex = 1;
          // 
          // myViewTabPage
          // 
-         this.myViewTabPage.Controls.Add(this.myDataGrid);
-         this.myViewTabPage.Location = new System.Drawing.Point(4, 26);
-         this.myViewTabPage.Name = "myViewTabPage";
-         this.myViewTabPage.Size = new System.Drawing.Size(896, 305);
-         this.myViewTabPage.TabIndex = 0;
-         this.myViewTabPage.Text = "View";
+         myViewTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myViewTabPage.Controls.Add(myDataGrid);
+         myViewTabPage.ForeColor = System.Drawing.Color.White;
+         myViewTabPage.Location = new System.Drawing.Point(4, 26);
+         myViewTabPage.Name = "myViewTabPage";
+         myViewTabPage.Size = new System.Drawing.Size(896, 305);
+         myViewTabPage.TabIndex = 0;
+         myViewTabPage.Text = "View";
          // 
          // myDataGrid
          // 
-         this.myDataGrid.BackgroundColor = System.Drawing.Color.Gainsboro;
-         this.myDataGrid.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
-         // this.myDataGrid.CaptionBackColor = System.Drawing.Color.Wheat;
-         // this.myDataGrid.CaptionFont = new System.Drawing.Font("Comic Sans MS", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         // this.myDataGrid.CaptionForeColor = System.Drawing.Color.Black;
-         this.myDataGrid.DataMember = "";
-         this.myDataGrid.Dock = System.Windows.Forms.DockStyle.Fill;
-         // this.myDataGrid.FlatMode = true;
-         // this.myDataGrid.GridLineColor = System.Drawing.Color.Gainsboro;
-         // this.myDataGrid.HeaderBackColor = System.Drawing.Color.Gainsboro;
-         // this.myDataGrid.HeaderFont = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         // this.myDataGrid.HeaderForeColor = System.Drawing.Color.Black;
-         this.myDataGrid.HorizScrollBarValue = 0;
-         this.myDataGrid.Location = new System.Drawing.Point(0, 0);
-         this.myDataGrid.Name = "myDataGrid";
-         // this.myDataGrid.PreferredRowHeight = 20;
-         this.myDataGrid.ReadOnly = true;
-         this.myDataGrid.RowHeadersVisible = false;
-         // this.myDataGrid.RowHeaderWidth = 0;
-         this.myDataGrid.Size = new System.Drawing.Size(896, 305);
-         this.myDataGrid.TabIndex = 0;
-         this.myDataGrid.MouseEnter += new System.EventHandler(this.myDataGrid_MouseEnter);
-         this.myDataGrid.MouseLeave += new System.EventHandler(this.myDataGrid_MouseLeave);
+         myDataGrid.BackgroundColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myDataGrid.ColumnHeadersBorderStyle = System.Windows.Forms.DataGridViewHeaderBorderStyle.Single;
+         myDataGrid.ColumnHeadersHeight = 32;
+         dataGridViewCellStyle1.Alignment = System.Windows.Forms.DataGridViewContentAlignment.MiddleLeft;
+         dataGridViewCellStyle1.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         dataGridViewCellStyle1.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         dataGridViewCellStyle1.ForeColor = System.Drawing.Color.White;
+         dataGridViewCellStyle1.SelectionBackColor = System.Drawing.Color.Green;
+         dataGridViewCellStyle1.SelectionForeColor = System.Drawing.Color.Yellow;
+         dataGridViewCellStyle1.WrapMode = System.Windows.Forms.DataGridViewTriState.False;
+         myDataGrid.DefaultCellStyle = dataGridViewCellStyle1;
+         myDataGrid.Dock = System.Windows.Forms.DockStyle.Fill;
+         myDataGrid.GridColor = System.Drawing.Color.DarkGray;
+         myDataGrid.HorizScrollBarValue = 0;
+         myDataGrid.Location = new System.Drawing.Point(0, 0);
+         myDataGrid.Name = "myDataGrid";
+         myDataGrid.ReadOnly = true;
+         myDataGrid.RowHeadersBorderStyle = System.Windows.Forms.DataGridViewHeaderBorderStyle.None;
+         dataGridViewCellStyle2.Alignment = System.Windows.Forms.DataGridViewContentAlignment.MiddleLeft;
+         dataGridViewCellStyle2.BackColor = System.Drawing.SystemColors.Control;
+         dataGridViewCellStyle2.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         dataGridViewCellStyle2.ForeColor = System.Drawing.Color.White;
+         dataGridViewCellStyle2.SelectionBackColor = System.Drawing.Color.Yellow;
+         dataGridViewCellStyle2.SelectionForeColor = System.Drawing.Color.DimGray;
+         dataGridViewCellStyle2.WrapMode = System.Windows.Forms.DataGridViewTriState.True;
+         myDataGrid.RowHeadersDefaultCellStyle = dataGridViewCellStyle2;
+         myDataGrid.RowHeadersVisible = false;
+         myDataGrid.RowHeadersWidth = 82;
+         myDataGrid.Size = new System.Drawing.Size(896, 305);
+         myDataGrid.TabIndex = 0;
+         myDataGrid.MouseEnter += myDataGrid_MouseEnter;
+         myDataGrid.MouseLeave += myDataGrid_MouseLeave;
          // 
          // myCSharpTabPage
          // 
-         this.myCSharpTabPage.Controls.Add(this.myCSharpText);
-         this.myCSharpTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myCSharpTabPage.Name = "myCSharpTabPage";
-         this.myCSharpTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myCSharpTabPage.TabIndex = 2;
-         this.myCSharpTabPage.Text = "C# Class";
+         myCSharpTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myCSharpTabPage.Controls.Add(myCSharpText);
+         myCSharpTabPage.ForeColor = System.Drawing.Color.White;
+         myCSharpTabPage.Location = new System.Drawing.Point(4, 27);
+         myCSharpTabPage.Name = "myCSharpTabPage";
+         myCSharpTabPage.Size = new System.Drawing.Size(896, 304);
+         myCSharpTabPage.TabIndex = 2;
+         myCSharpTabPage.Text = "C# Class";
          // 
          // myCSharpText
          // 
-         this.myCSharpText.BackColor = System.Drawing.Color.White;
-         this.myCSharpText.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myCSharpText.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myCSharpText.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         this.myCSharpText.Location = new System.Drawing.Point(0, 0);
-         this.myCSharpText.Name = "myCSharpText";
-         this.myCSharpText.ReadOnly = true;
-         this.myCSharpText.Size = new System.Drawing.Size(896, 306);
-         this.myCSharpText.TabIndex = 0;
-         this.myCSharpText.Text = "";
-         this.myCSharpText.MouseEnter += new System.EventHandler(this.myCSharpText_MouseEnter);
-         this.myCSharpText.MouseLeave += new System.EventHandler(this.myCSharpText_MouseLeave);
+         myCSharpText.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myCSharpText.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myCSharpText.Dock = System.Windows.Forms.DockStyle.Fill;
+         myCSharpText.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         myCSharpText.ForeColor = System.Drawing.Color.White;
+         myCSharpText.Location = new System.Drawing.Point(0, 0);
+         myCSharpText.Name = "myCSharpText";
+         myCSharpText.ReadOnly = true;
+         myCSharpText.Size = new System.Drawing.Size(896, 304);
+         myCSharpText.TabIndex = 0;
+         myCSharpText.Text = "";
+         myCSharpText.MouseEnter += myCSharpText_MouseEnter;
+         myCSharpText.MouseLeave += myCSharpText_MouseLeave;
          // 
          // myErrorTabPage
          // 
-         this.myErrorTabPage.Controls.Add(this.myErrorTextBox);
-         this.myErrorTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myErrorTabPage.Name = "myErrorTabPage";
-         this.myErrorTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myErrorTabPage.TabIndex = 1;
-         this.myErrorTabPage.Text = "Error";
+         myErrorTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myErrorTabPage.Controls.Add(myErrorTextBox);
+         myErrorTabPage.ForeColor = System.Drawing.Color.White;
+         myErrorTabPage.Location = new System.Drawing.Point(4, 27);
+         myErrorTabPage.Name = "myErrorTabPage";
+         myErrorTabPage.Size = new System.Drawing.Size(896, 304);
+         myErrorTabPage.TabIndex = 1;
+         myErrorTabPage.Text = "Error";
          // 
          // myErrorTextBox
          // 
-         this.myErrorTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myErrorTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myErrorTextBox.ForeColor = System.Drawing.Color.Red;
-         this.myErrorTextBox.Location = new System.Drawing.Point(0, 0);
-         this.myErrorTextBox.Name = "myErrorTextBox";
-         this.myErrorTextBox.ReadOnly = true;
-         this.myErrorTextBox.Size = new System.Drawing.Size(896, 306);
-         this.myErrorTextBox.TabIndex = 1;
-         this.myErrorTextBox.Text = "";
+         myErrorTextBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myErrorTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myErrorTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
+         myErrorTextBox.ForeColor = System.Drawing.Color.Red;
+         myErrorTextBox.Location = new System.Drawing.Point(0, 0);
+         myErrorTextBox.Name = "myErrorTextBox";
+         myErrorTextBox.ReadOnly = true;
+         myErrorTextBox.Size = new System.Drawing.Size(896, 304);
+         myErrorTextBox.TabIndex = 1;
+         myErrorTextBox.Text = "";
          // 
          // myTableListTabPage
          // 
-         this.myTableListTabPage.Controls.Add(this.myTableListBox);
-         this.myTableListTabPage.Controls.Add(this.myTableListTopPanel);
-         this.myTableListTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myTableListTabPage.Name = "myTableListTabPage";
-         this.myTableListTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myTableListTabPage.TabIndex = 3;
-         this.myTableListTabPage.Text = "Table list";
+         myTableListTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableListTabPage.Controls.Add(myTableListBox);
+         myTableListTabPage.Controls.Add(myTableListTopPanel);
+         myTableListTabPage.ForeColor = System.Drawing.Color.White;
+         myTableListTabPage.Location = new System.Drawing.Point(4, 27);
+         myTableListTabPage.Name = "myTableListTabPage";
+         myTableListTabPage.Size = new System.Drawing.Size(896, 304);
+         myTableListTabPage.TabIndex = 3;
+         myTableListTabPage.Text = "Table list";
          // 
          // myTableListBox
          // 
-         this.myTableListBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myTableListBox.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myTableListBox.DrawMode = System.Windows.Forms.DrawMode.OwnerDrawFixed;
-         this.myTableListBox.ItemHeight = 14;
-         this.myTableListBox.Location = new System.Drawing.Point(0, 31);
-         this.myTableListBox.Name = "myTableListBox";
-         this.myTableListBox.Size = new System.Drawing.Size(896, 275);
-         this.myTableListBox.TabIndex = 1;
-         this.myTableListBox.DoubleClick += new System.EventHandler(this.myTableListBox_DoubleClick);
-         this.myTableListBox.MouseEnter += new System.EventHandler(this.myTableListBox_MouseEnter);
-         this.myTableListBox.MouseLeave += new System.EventHandler(this.myTableListBox_MouseLeave);
+         myTableListBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableListBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myTableListBox.Dock = System.Windows.Forms.DockStyle.Fill;
+         myTableListBox.DrawMode = System.Windows.Forms.DrawMode.OwnerDrawFixed;
+         myTableListBox.ForeColor = System.Drawing.Color.White;
+         myTableListBox.ItemHeight = 14;
+         myTableListBox.Location = new System.Drawing.Point(0, 31);
+         myTableListBox.Name = "myTableListBox";
+         myTableListBox.Size = new System.Drawing.Size(896, 273);
+         myTableListBox.TabIndex = 1;
+         myTableListBox.DoubleClick += myTableListBox_DoubleClick;
+         myTableListBox.MouseEnter += myTableListBox_MouseEnter;
+         myTableListBox.MouseLeave += myTableListBox_MouseLeave;
          // 
          // myTableListTopPanel
          // 
-         this.myTableListTopPanel.Controls.Add(this.myTableFilterBox);
-         this.myTableListTopPanel.Controls.Add(this.myTableFilterLabel);
-         this.myTableListTopPanel.Dock = System.Windows.Forms.DockStyle.Top;
-         this.myTableListTopPanel.Location = new System.Drawing.Point(0, 0);
-         this.myTableListTopPanel.Name = "myTableListTopPanel";
-         this.myTableListTopPanel.Size = new System.Drawing.Size(896, 31);
-         this.myTableListTopPanel.TabIndex = 4;
+         myTableListTopPanel.Controls.Add(myTableFilterBox);
+         myTableListTopPanel.Controls.Add(myTableFilterLabel);
+         myTableListTopPanel.Dock = System.Windows.Forms.DockStyle.Top;
+         myTableListTopPanel.Location = new System.Drawing.Point(0, 0);
+         myTableListTopPanel.Name = "myTableListTopPanel";
+         myTableListTopPanel.Size = new System.Drawing.Size(896, 31);
+         myTableListTopPanel.TabIndex = 4;
          // 
          // myTableFilterBox
          // 
-         this.myTableFilterBox.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
-            | System.Windows.Forms.AnchorStyles.Left) 
-            | System.Windows.Forms.AnchorStyles.Right)));
-         this.myTableFilterBox.Location = new System.Drawing.Point(58, 3);
-         this.myTableFilterBox.Name = "myTableFilterBox";
-         this.myTableFilterBox.Size = new System.Drawing.Size(835, 20);
-         this.myTableFilterBox.TabIndex = 2;
-         this.myTableFilterBox.TextChanged += new System.EventHandler(this.myTableFilterBox_TextChanged);
+         myTableFilterBox.Anchor = System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right;
+         myTableFilterBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableFilterBox.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
+         myTableFilterBox.ForeColor = System.Drawing.Color.White;
+         myTableFilterBox.Location = new System.Drawing.Point(58, 3);
+         myTableFilterBox.Name = "myTableFilterBox";
+         myTableFilterBox.Size = new System.Drawing.Size(835, 20);
+         myTableFilterBox.TabIndex = 2;
+         myTableFilterBox.TextChanged += myTableFilterBox_TextChanged;
          // 
          // myTableFilterLabel
          // 
-         this.myTableFilterLabel.AutoSize = true;
-         this.myTableFilterLabel.Location = new System.Drawing.Point(3, 3);
-         this.myTableFilterLabel.Name = "myTableFilterLabel";
-         this.myTableFilterLabel.Size = new System.Drawing.Size(49, 14);
-         this.myTableFilterLabel.TabIndex = 3;
-         this.myTableFilterLabel.Text = "Filter";
+         myTableFilterLabel.AutoSize = true;
+         myTableFilterLabel.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableFilterLabel.ForeColor = System.Drawing.Color.White;
+         myTableFilterLabel.Location = new System.Drawing.Point(3, 3);
+         myTableFilterLabel.Name = "myTableFilterLabel";
+         myTableFilterLabel.Size = new System.Drawing.Size(49, 14);
+         myTableFilterLabel.TabIndex = 3;
+         myTableFilterLabel.Text = "Filter";
+         myTableFilterLabel.Click += myTableFilterLabel_Click;
          // 
          // myPrimitivesPage
          // 
-         this.myPrimitivesPage.Controls.Add(this.myTreeView);
-         this.myPrimitivesPage.Location = new System.Drawing.Point(4, 25);
-         this.myPrimitivesPage.Name = "myPrimitivesPage";
-         this.myPrimitivesPage.Size = new System.Drawing.Size(896, 306);
-         this.myPrimitivesPage.TabIndex = 4;
-         this.myPrimitivesPage.Text = "Language primitives";
+         myPrimitivesPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myPrimitivesPage.Controls.Add(myTreeView);
+         myPrimitivesPage.ForeColor = System.Drawing.Color.White;
+         myPrimitivesPage.Location = new System.Drawing.Point(4, 27);
+         myPrimitivesPage.Name = "myPrimitivesPage";
+         myPrimitivesPage.Size = new System.Drawing.Size(896, 304);
+         myPrimitivesPage.TabIndex = 4;
+         myPrimitivesPage.Text = "Language primitives";
          // 
          // myTreeView
          // 
-         this.myTreeView.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myTreeView.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myTreeView.Location = new System.Drawing.Point(0, 0);
-         this.myTreeView.Name = "myTreeView";
-         this.myTreeView.Size = new System.Drawing.Size(896, 306);
-         this.myTreeView.TabIndex = 0;
+         myTreeView.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTreeView.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myTreeView.Dock = System.Windows.Forms.DockStyle.Fill;
+         myTreeView.ForeColor = System.Drawing.Color.White;
+         myTreeView.Location = new System.Drawing.Point(0, 0);
+         myTreeView.Name = "myTreeView";
+         myTreeView.Size = new System.Drawing.Size(896, 304);
+         myTreeView.TabIndex = 0;
          // 
          // myConsoleTabPage
          // 
-         this.myConsoleTabPage.Controls.Add(this.myConsoleRichTextBox);
-         this.myConsoleTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myConsoleTabPage.Name = "myConsoleTabPage";
-         this.myConsoleTabPage.Padding = new System.Windows.Forms.Padding(3);
-         this.myConsoleTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myConsoleTabPage.TabIndex = 5;
-         this.myConsoleTabPage.Text = "Console";
-         this.myConsoleTabPage.UseVisualStyleBackColor = true;
+         myConsoleTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myConsoleTabPage.Controls.Add(myConsoleRichTextBox);
+         myConsoleTabPage.ForeColor = System.Drawing.Color.White;
+         myConsoleTabPage.Location = new System.Drawing.Point(4, 27);
+         myConsoleTabPage.Name = "myConsoleTabPage";
+         myConsoleTabPage.Padding = new System.Windows.Forms.Padding(3);
+         myConsoleTabPage.Size = new System.Drawing.Size(896, 304);
+         myConsoleTabPage.TabIndex = 5;
+         myConsoleTabPage.Text = "Console";
          // 
          // myConsoleRichTextBox
          // 
-         this.myConsoleRichTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myConsoleRichTextBox.Location = new System.Drawing.Point(3, 3);
-         this.myConsoleRichTextBox.Name = "myConsoleRichTextBox";
-         this.myConsoleRichTextBox.Size = new System.Drawing.Size(890, 300);
-         this.myConsoleRichTextBox.TabIndex = 0;
-         this.myConsoleRichTextBox.Text = "";
+         myConsoleRichTextBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myConsoleRichTextBox.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
+         myConsoleRichTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
+         myConsoleRichTextBox.ForeColor = System.Drawing.Color.White;
+         myConsoleRichTextBox.Location = new System.Drawing.Point(3, 3);
+         myConsoleRichTextBox.Name = "myConsoleRichTextBox";
+         myConsoleRichTextBox.Size = new System.Drawing.Size(890, 298);
+         myConsoleRichTextBox.TabIndex = 0;
+         myConsoleRichTextBox.Text = "";
          // 
          // myTimer
          // 
-         this.myTimer.Enabled = true;
-         this.myTimer.Interval = 150;
-         this.myTimer.Tick += new System.EventHandler(this.myTimer_Tick);
+         myTimer.Enabled = true;
+         myTimer.Interval = 150;
+         myTimer.Tick += myTimer_Tick;
          // 
          // mySplitter
          // 
-         this.mySplitter.Dock = System.Windows.Forms.DockStyle.Top;
-         this.mySplitter.Location = new System.Drawing.Point(0, 232);
-         this.mySplitter.Name = "mySplitter";
-         this.mySplitter.Size = new System.Drawing.Size(904, 3);
-         this.mySplitter.TabIndex = 3;
-         this.mySplitter.TabStop = false;
+         mySplitter.Dock = System.Windows.Forms.DockStyle.Top;
+         mySplitter.Location = new System.Drawing.Point(0, 232);
+         mySplitter.Name = "mySplitter";
+         mySplitter.Size = new System.Drawing.Size(904, 4);
+         mySplitter.TabIndex = 3;
+         mySplitter.TabStop = false;
          // 
          // myStatusStrip
          // 
-         this.myStatusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripStatusLabelTime,
-            this.myToolStripStatusLabelRows});
-         this.myStatusStrip.Location = new System.Drawing.Point(0, 567);
-         this.myStatusStrip.Name = "myStatusStrip";
-         this.myStatusStrip.Size = new System.Drawing.Size(904, 22);
-         this.myStatusStrip.TabIndex = 4;
+         myStatusStrip.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myStatusStrip.ImageScalingSize = new System.Drawing.Size(32, 32);
+         myStatusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripStatusLabelTime, myToolStripStatusLabelRows, toolStripStatusLabel1 });
+         myStatusStrip.Location = new System.Drawing.Point(0, 567);
+         myStatusStrip.Name = "myStatusStrip";
+         myStatusStrip.Size = new System.Drawing.Size(904, 22);
+         myStatusStrip.TabIndex = 4;
          // 
          // myToolStripStatusLabelTime
          // 
-         this.myToolStripStatusLabelTime.Name = "myToolStripStatusLabelTime";
-         this.myToolStripStatusLabelTime.Padding = new System.Windows.Forms.Padding(0, 0, 8, 0);
-         this.myToolStripStatusLabelTime.Size = new System.Drawing.Size(8, 17);
+         myToolStripStatusLabelTime.Name = "myToolStripStatusLabelTime";
+         myToolStripStatusLabelTime.Padding = new System.Windows.Forms.Padding(0, 0, 8, 0);
+         myToolStripStatusLabelTime.Size = new System.Drawing.Size(8, 17);
          // 
          // myToolStripStatusLabelRows
          // 
-         this.myToolStripStatusLabelRows.Name = "myToolStripStatusLabelRows";
-         this.myToolStripStatusLabelRows.Size = new System.Drawing.Size(0, 17);
+         myToolStripStatusLabelRows.Name = "myToolStripStatusLabelRows";
+         myToolStripStatusLabelRows.Size = new System.Drawing.Size(0, 17);
+         // 
+         // toolStripStatusLabel1
+         // 
+         toolStripStatusLabel1.ForeColor = System.Drawing.Color.White;
+         toolStripStatusLabel1.Name = "toolStripStatusLabel1";
+         toolStripStatusLabel1.Size = new System.Drawing.Size(0, 17);
          // 
          // myMenuStrip
          // 
-         this.myMenuStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemFile,
-            this.myToolStripMenuItemQuery,
-            this.myToolStripMenuItemTools,
-            this.myToolStripMenuItemSite,
-            this.libraryToolStripMenuItem});
-         this.myMenuStrip.Location = new System.Drawing.Point(0, 0);
-         this.myMenuStrip.Name = "myMenuStrip";
-         this.myMenuStrip.Size = new System.Drawing.Size(904, 24);
-         this.myMenuStrip.TabIndex = 5;
-         this.myMenuStrip.Text = "menuStrip1";
+         myMenuStrip.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myMenuStrip.ImageScalingSize = new System.Drawing.Size(32, 32);
+         myMenuStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemFile, myToolStripMenuItemQuery, myToolStripMenuItemTools, myToolStripMenuItemSite, libraryToolStripMenuItem });
+         myMenuStrip.Location = new System.Drawing.Point(0, 0);
+         myMenuStrip.Name = "myMenuStrip";
+         myMenuStrip.Size = new System.Drawing.Size(904, 24);
+         myMenuStrip.TabIndex = 5;
+         myMenuStrip.Text = "menuStrip1";
          // 
          // myToolStripMenuItemFile
          // 
-         this.myToolStripMenuItemFile.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemOpen,
-            this.myToolStripMenuItemSave,
-            this.myToolStripMenuItemSaveAs,
-            this.myToolStripMenuItemExit,
-            this.testGridToolStripMenuItem});
-         this.myToolStripMenuItemFile.Name = "myToolStripMenuItemFile";
-         this.myToolStripMenuItemFile.Size = new System.Drawing.Size(37, 20);
-         this.myToolStripMenuItemFile.Text = "&File";
+         myToolStripMenuItemFile.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemOpen, myToolStripMenuItemSave, myToolStripMenuItemSaveAs, myToolStripMenuItemExit, testGridToolStripMenuItem });
+         myToolStripMenuItemFile.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemFile.Name = "myToolStripMenuItemFile";
+         myToolStripMenuItemFile.Size = new System.Drawing.Size(37, 20);
+         myToolStripMenuItemFile.Text = "&File";
          // 
          // myToolStripMenuItemOpen
          // 
-         this.myToolStripMenuItemOpen.Image = global::DataStreamQueryAnalyzer.Properties.Resources.File_Open_icon16;
-         this.myToolStripMenuItemOpen.Name = "myToolStripMenuItemOpen";
-         this.myToolStripMenuItemOpen.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemOpen.Text = "&Open...";
-         this.myToolStripMenuItemOpen.Click += new System.EventHandler(this.myMenuItemOpenClick);
+         myToolStripMenuItemOpen.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemOpen.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemOpen.Image = DataStreamQueryAnalyzer.Properties.Resources.File_Open_icon16;
+         myToolStripMenuItemOpen.Name = "myToolStripMenuItemOpen";
+         myToolStripMenuItemOpen.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemOpen.Text = "&Open...";
+         myToolStripMenuItemOpen.Click += myMenuItemOpenClick;
          // 
          // myToolStripMenuItemSave
          // 
-         this.myToolStripMenuItemSave.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Save_icon16;
-         this.myToolStripMenuItemSave.Name = "myToolStripMenuItemSave";
-         this.myToolStripMenuItemSave.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemSave.Text = "Save...";
-         this.myToolStripMenuItemSave.Click += new System.EventHandler(this.myMenuItemSaveClick);
+         myToolStripMenuItemSave.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemSave.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemSave.Image = DataStreamQueryAnalyzer.Properties.Resources.Save_icon16;
+         myToolStripMenuItemSave.Name = "myToolStripMenuItemSave";
+         myToolStripMenuItemSave.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemSave.Text = "Save...";
+         myToolStripMenuItemSave.Click += myMenuItemSaveClick;
          // 
          // myToolStripMenuItemSaveAs
          // 
-         this.myToolStripMenuItemSaveAs.Name = "myToolStripMenuItemSaveAs";
-         this.myToolStripMenuItemSaveAs.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemSaveAs.Text = "Save As...";
-         this.myToolStripMenuItemSaveAs.Click += new System.EventHandler(this.myMenuItemSaveAsClick);
+         myToolStripMenuItemSaveAs.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemSaveAs.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemSaveAs.Name = "myToolStripMenuItemSaveAs";
+         myToolStripMenuItemSaveAs.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemSaveAs.Text = "Save As...";
+         myToolStripMenuItemSaveAs.Click += myMenuItemSaveAsClick;
          // 
          // myToolStripMenuItemExit
          // 
-         this.myToolStripMenuItemExit.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Turn_Off_icon16;
-         this.myToolStripMenuItemExit.Name = "myToolStripMenuItemExit";
-         this.myToolStripMenuItemExit.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemExit.Text = "Exit";
-         this.myToolStripMenuItemExit.Click += new System.EventHandler(this.myToolStripMenuItemExit_Click);
+         myToolStripMenuItemExit.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemExit.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemExit.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Turn_Off_icon16;
+         myToolStripMenuItemExit.Name = "myToolStripMenuItemExit";
+         myToolStripMenuItemExit.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemExit.Text = "Exit";
+         myToolStripMenuItemExit.Click += myToolStripMenuItemExit_Click;
          // 
          // testGridToolStripMenuItem
          // 
-         this.testGridToolStripMenuItem.Name = "testGridToolStripMenuItem";
-         this.testGridToolStripMenuItem.Size = new System.Drawing.Size(123, 22);
-         this.testGridToolStripMenuItem.Text = "Test Grid";
-         this.testGridToolStripMenuItem.Click += new System.EventHandler(this.testGridToolStripMenuItem_Click);
+         testGridToolStripMenuItem.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         testGridToolStripMenuItem.ForeColor = System.Drawing.Color.White;
+         testGridToolStripMenuItem.Name = "testGridToolStripMenuItem";
+         testGridToolStripMenuItem.Size = new System.Drawing.Size(139, 38);
+         testGridToolStripMenuItem.Text = "Test Grid";
+         testGridToolStripMenuItem.Click += testGridToolStripMenuItem_Click;
          // 
          // myToolStripMenuItemQuery
          // 
-         this.myToolStripMenuItemQuery.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemExecute,
-            this.myToolStripMenuItemExecSel,
-            this.myToolStripMenuItemFreeze,
-            this.myToolStripMenuItemStep});
-         this.myToolStripMenuItemQuery.Name = "myToolStripMenuItemQuery";
-         this.myToolStripMenuItemQuery.Size = new System.Drawing.Size(51, 20);
-         this.myToolStripMenuItemQuery.Text = "&Query";
+         myToolStripMenuItemQuery.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemExecute, myToolStripMenuItemExecSel, myToolStripMenuItemFreeze, myToolStripMenuItemStep });
+         myToolStripMenuItemQuery.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemQuery.Name = "myToolStripMenuItemQuery";
+         myToolStripMenuItemQuery.Size = new System.Drawing.Size(51, 20);
+         myToolStripMenuItemQuery.Text = "&Query";
          // 
          // myToolStripMenuItemExecute
          // 
-         this.myToolStripMenuItemExecute.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Play_icon16;
-         this.myToolStripMenuItemExecute.Name = "myToolStripMenuItemExecute";
-         this.myToolStripMenuItemExecute.ShortcutKeys = System.Windows.Forms.Keys.F11;
-         this.myToolStripMenuItemExecute.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemExecute.Text = "Execute";
-         this.myToolStripMenuItemExecute.Click += new System.EventHandler(this.myMenuItemExecuteQueryClick);
+         myToolStripMenuItemExecute.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemExecute.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemExecute.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Play_icon16;
+         myToolStripMenuItemExecute.Name = "myToolStripMenuItemExecute";
+         myToolStripMenuItemExecute.ShortcutKeys = System.Windows.Forms.Keys.F11;
+         myToolStripMenuItemExecute.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemExecute.Text = "Execute";
+         myToolStripMenuItemExecute.Click += myMenuItemExecuteQueryClick;
          // 
          // myToolStripMenuItemExecSel
          // 
-         this.myToolStripMenuItemExecSel.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Blank_Blue_icon16;
-         this.myToolStripMenuItemExecSel.Name = "myToolStripMenuItemExecSel";
-         this.myToolStripMenuItemExecSel.ShortcutKeys = System.Windows.Forms.Keys.F12;
-         this.myToolStripMenuItemExecSel.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemExecSel.Text = "Execute Selection";
-         this.myToolStripMenuItemExecSel.Click += new System.EventHandler(this.myToolStripMenuItemExecSel_Click);
+         myToolStripMenuItemExecSel.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemExecSel.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemExecSel.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Blank_Blue_icon16;
+         myToolStripMenuItemExecSel.Name = "myToolStripMenuItemExecSel";
+         myToolStripMenuItemExecSel.ShortcutKeys = System.Windows.Forms.Keys.F12;
+         myToolStripMenuItemExecSel.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemExecSel.Text = "Execute Selection";
+         myToolStripMenuItemExecSel.Click += myToolStripMenuItemExecSel_Click;
          // 
          // myToolStripMenuItemFreeze
          // 
-         this.myToolStripMenuItemFreeze.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Pause_icon16;
-         this.myToolStripMenuItemFreeze.Name = "myToolStripMenuItemFreeze";
-         this.myToolStripMenuItemFreeze.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemFreeze.Text = "Freeze";
-         this.myToolStripMenuItemFreeze.Click += new System.EventHandler(this.myToolStripMenuItemFreeze_Click);
+         myToolStripMenuItemFreeze.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemFreeze.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemFreeze.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Pause_icon16;
+         myToolStripMenuItemFreeze.Name = "myToolStripMenuItemFreeze";
+         myToolStripMenuItemFreeze.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemFreeze.Text = "Freeze";
+         myToolStripMenuItemFreeze.Click += myToolStripMenuItemFreeze_Click;
          // 
          // myToolStripMenuItemStep
          // 
-         this.myToolStripMenuItemStep.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Forward_icon16;
-         this.myToolStripMenuItemStep.Name = "myToolStripMenuItemStep";
-         this.myToolStripMenuItemStep.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemStep.Text = "Step";
-         this.myToolStripMenuItemStep.Click += new System.EventHandler(this.myToolStripMenuItemStep_Click);
+         myToolStripMenuItemStep.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemStep.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemStep.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Forward_icon16;
+         myToolStripMenuItemStep.Name = "myToolStripMenuItemStep";
+         myToolStripMenuItemStep.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemStep.Text = "Step";
+         myToolStripMenuItemStep.Click += myToolStripMenuItemStep_Click;
          // 
          // myToolStripMenuItemTools
          // 
-         this.myToolStripMenuItemTools.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemLoadAss,
-            this.myToolStripMenuItemTest,
-            this.myToolStripMenuItemToolsSep,
-            this.myToolStripMenuItemClearConsole});
-         this.myToolStripMenuItemTools.Name = "myToolStripMenuItemTools";
-         this.myToolStripMenuItemTools.Size = new System.Drawing.Size(48, 20);
-         this.myToolStripMenuItemTools.Text = "&Tools";
+         myToolStripMenuItemTools.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemLoadAss, myToolStripMenuItemTest, myToolStripMenuItemToolsSep, myToolStripMenuItemClearConsole });
+         myToolStripMenuItemTools.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemTools.Name = "myToolStripMenuItemTools";
+         myToolStripMenuItemTools.Size = new System.Drawing.Size(46, 20);
+         myToolStripMenuItemTools.Text = "&Tools";
          // 
          // myToolStripMenuItemLoadAss
          // 
-         this.myToolStripMenuItemLoadAss.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Upload_icon16;
-         this.myToolStripMenuItemLoadAss.Name = "myToolStripMenuItemLoadAss";
-         this.myToolStripMenuItemLoadAss.Size = new System.Drawing.Size(154, 22);
-         this.myToolStripMenuItemLoadAss.Text = "Load Assembly";
-         this.myToolStripMenuItemLoadAss.Click += new System.EventHandler(this.myToolStripMenuItemLoadAss_Click);
+         myToolStripMenuItemLoadAss.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Upload_icon16;
+         myToolStripMenuItemLoadAss.Name = "myToolStripMenuItemLoadAss";
+         myToolStripMenuItemLoadAss.Size = new System.Drawing.Size(170, 38);
+         myToolStripMenuItemLoadAss.Text = "Load Assembly";
+         myToolStripMenuItemLoadAss.Click += myToolStripMenuItemLoadAss_Click;
          // 
          // myToolStripMenuItemTest
          // 
-         this.myToolStripMenuItemTest.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Refresh_icon16;
-         this.myToolStripMenuItemTest.Name = "myToolStripMenuItemTest";
-         this.myToolStripMenuItemTest.ShortcutKeys = System.Windows.Forms.Keys.F4;
-         this.myToolStripMenuItemTest.Size = new System.Drawing.Size(154, 22);
-         this.myToolStripMenuItemTest.Text = "Test Resub";
-         this.myToolStripMenuItemTest.Click += new System.EventHandler(this.myToolStripMenuItemTest_Click);
+         myToolStripMenuItemTest.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Refresh_icon16;
+         myToolStripMenuItemTest.Name = "myToolStripMenuItemTest";
+         myToolStripMenuItemTest.ShortcutKeys = System.Windows.Forms.Keys.F4;
+         myToolStripMenuItemTest.Size = new System.Drawing.Size(170, 38);
+         myToolStripMenuItemTest.Text = "Test Resub";
+         myToolStripMenuItemTest.Click += myToolStripMenuItemTest_Click;
          // 
          // myToolStripMenuItemToolsSep
          // 
-         this.myToolStripMenuItemToolsSep.Name = "myToolStripMenuItemToolsSep";
-         this.myToolStripMenuItemToolsSep.Size = new System.Drawing.Size(151, 6);
+         myToolStripMenuItemToolsSep.Name = "myToolStripMenuItemToolsSep";
+         myToolStripMenuItemToolsSep.Size = new System.Drawing.Size(167, 6);
          // 
          // myToolStripMenuItemClearConsole
          // 
-         this.myToolStripMenuItemClearConsole.Name = "myToolStripMenuItemClearConsole";
-         this.myToolStripMenuItemClearConsole.Size = new System.Drawing.Size(154, 22);
-         this.myToolStripMenuItemClearConsole.Text = "Clear Console";
-         this.myToolStripMenuItemClearConsole.Click += new System.EventHandler(this.myToolStripMenuItemClearConsole_Click);
+         myToolStripMenuItemClearConsole.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemClearConsole.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemClearConsole.Name = "myToolStripMenuItemClearConsole";
+         myToolStripMenuItemClearConsole.Size = new System.Drawing.Size(170, 38);
+         myToolStripMenuItemClearConsole.Text = "Clear Console";
+         myToolStripMenuItemClearConsole.Click += myToolStripMenuItemClearConsole_Click;
          // 
          // myToolStripMenuItemSite
          // 
-         this.myToolStripMenuItemSite.Name = "myToolStripMenuItemSite";
-         this.myToolStripMenuItemSite.Size = new System.Drawing.Size(38, 20);
-         this.myToolStripMenuItemSite.Text = "&Site";
+         myToolStripMenuItemSite.Name = "myToolStripMenuItemSite";
+         myToolStripMenuItemSite.Size = new System.Drawing.Size(38, 20);
+         myToolStripMenuItemSite.Text = "&Site";
          // 
          // libraryToolStripMenuItem
          // 
-         this.libraryToolStripMenuItem.Name = "libraryToolStripMenuItem";
-         this.libraryToolStripMenuItem.Size = new System.Drawing.Size(55, 20);
-         this.libraryToolStripMenuItem.Text = "Library";
-         this.libraryToolStripMenuItem.Click += new System.EventHandler(this.libraryToolStripMenuItem_Click);
+         libraryToolStripMenuItem.Name = "libraryToolStripMenuItem";
+         libraryToolStripMenuItem.Size = new System.Drawing.Size(55, 20);
+         libraryToolStripMenuItem.Text = "Library";
+         libraryToolStripMenuItem.Click += libraryToolStripMenuItem_Click;
          // 
          // MainForm
          // 
-         this.AutoScaleBaseSize = new System.Drawing.Size(7, 13);
-         this.BackColor = System.Drawing.Color.Gainsboro;
-         this.ClientSize = new System.Drawing.Size(904, 589);
-         this.Controls.Add(this.mySplitter);
-         this.Controls.Add(this.myTabControl);
-         this.Controls.Add(this.myQueryTextBox);
-         this.Controls.Add(this.myMenuStrip);
-         this.Controls.Add(this.myStatusStrip);
-         this.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         this.ForeColor = System.Drawing.Color.Black;
-         this.Icon = ((System.Drawing.Icon)(resources.GetObject("$this.Icon")));
-         this.MainMenuStrip = this.myMenuStrip;
-         this.Name = "MainForm";
-         this.StartPosition = System.Windows.Forms.FormStartPosition.Manual;
-         this.Text = "DataStream Query Analyzer";
-         this.Closed += new System.EventHandler(this.MainForm_Closed);
-         this.myTabControl.ResumeLayout(false);
-         this.myViewTabPage.ResumeLayout(false);
-         ((System.ComponentModel.ISupportInitialize)(this.myDataGrid)).EndInit();
-         this.myCSharpTabPage.ResumeLayout(false);
-         this.myErrorTabPage.ResumeLayout(false);
-         this.myTableListTabPage.ResumeLayout(false);
-         this.myTableListTopPanel.ResumeLayout(false);
-         this.myTableListTopPanel.PerformLayout();
-         this.myPrimitivesPage.ResumeLayout(false);
-         this.myConsoleTabPage.ResumeLayout(false);
-         this.myStatusStrip.ResumeLayout(false);
-         this.myStatusStrip.PerformLayout();
-         this.myMenuStrip.ResumeLayout(false);
-         this.myMenuStrip.PerformLayout();
-         this.ResumeLayout(false);
-         this.PerformLayout();
-
+         AutoScaleBaseSize = new System.Drawing.Size(7, 13);
+         BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         ClientSize = new System.Drawing.Size(904, 589);
+         Controls.Add(mySplitter);
+         Controls.Add(myTabControl);
+         Controls.Add(myQueryTextBox);
+         Controls.Add(myMenuStrip);
+         Controls.Add(myStatusStrip);
+         Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         ForeColor = System.Drawing.Color.White;
+         Icon = (System.Drawing.Icon)resources.GetObject("$this.Icon");
+         MainMenuStrip = myMenuStrip;
+         Name = "MainForm";
+         StartPosition = System.Windows.Forms.FormStartPosition.Manual;
+         Text = "BTS-cloud.io";
+         Closed += MainForm_Closed;
+         Load += MainForm_Load;
+         myTabControl.ResumeLayout(false);
+         myViewTabPage.ResumeLayout(false);
+         ((System.ComponentModel.ISupportInitialize)myDataGrid).EndInit();
+         myCSharpTabPage.ResumeLayout(false);
+         myErrorTabPage.ResumeLayout(false);
+         myTableListTabPage.ResumeLayout(false);
+         myTableListTopPanel.ResumeLayout(false);
+         myTableListTopPanel.PerformLayout();
+         myPrimitivesPage.ResumeLayout(false);
+         myConsoleTabPage.ResumeLayout(false);
+         myStatusStrip.ResumeLayout(false);
+         myStatusStrip.PerformLayout();
+         myMenuStrip.ResumeLayout(false);
+         myMenuStrip.PerformLayout();
+         ResumeLayout(false);
+         PerformLayout();
       }
       #endregion
 
@@ -561,5 +610,6 @@
       private System.Windows.Forms.Panel myTableListTopPanel;
       private System.Windows.Forms.ToolStripMenuItem testGridToolStripMenuItem;
       private System.Windows.Forms.ToolStripMenuItem libraryToolStripMenuItem;
+      private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel1;
    }
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -21,55 +21,65 @@ using Spark.DataStream;
 using Bts.DataStreamExpressions;
 // using BCG.Evaluator.DataStreamExpression;
 // using BCG.Utilities.Win32;
+using System.Threading.Tasks;
 
-namespace BCG
-{
-   public partial class MainForm : System.Windows.Forms.Form
-   {
+
+
+namespace BCG {
+   public partial class MainForm : System.Windows.Forms.Form {
+      // static async Task _run() {
+      //    var login = "bts";
+      //    var passwd = String.Empty;
+      //    Console.WriteLine("Starting DSQA... ");
+      //    // Log.notnull(login);
+      //    // Log.notnull(passwd);
+      //    var service_ctx = bts.api.core.ServiceContext.create();
+      //    await service_ctx.login(login, passwd);
+      // }
       private bool myChangeStyle = false;
       private PubSubRoute[] myRoutes = PubSub3Config.instance.getSubRouting();
       private PersistWindowState myWindowState;
       private string myLastFileName;
       private ViewStream myCurrentStream;
-      private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem> ();
+      private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
       // private string mySite = SitesCore.getMySiteName ();
       private string mySite = "TODO";
       private string myLastClassName = "MyView";
-      private ViewStream myAllTagsStream; 
-      private PubSubConfigureForm myPubSubConForm = new PubSubConfigureForm ();
+      private ViewStream myAllTagsStream;
+      private PubSubConfigureForm myPubSubConForm = new PubSubConfigureForm();
       private string myKQ;
       private TextWriter myConsoleOutput;
-      
-      public MainForm()
-      {        
+
+      public MainForm() {
+         //Task.Run(async () => await _run());
          InitializeComponent();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
          myKQ = "TODO";
          mySetText();
 
-         myWindowState = new PersistWindowState ();
+         myWindowState = new PersistWindowState();
          myWindowState.parent = this;
          myWindowState.saveState = false;
 
-         this.Load += new EventHandler (myLoad);
-         this.Closing += new CancelEventHandler (myClosing);         
+         this.Load += new EventHandler(myLoad);
+         this.Closing += new CancelEventHandler(myClosing);
+         myQueryTextBox.TextChanged += (o, e) => {
+            myQueryTextBox.ForeColor = Color.White;
+         };
       }
 
-      public string query_text
-      {
+      public string query_text {
          get { return myQueryTextBox.Text; }
          set { myQueryTextBox.Text = value; }
       }
-      private ViewStream myGetAllTagsStream ()
-      {
+      private ViewStream myGetAllTagsStream() {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
-         // return new FilterStream(new PublishedTagsStream(), String.Format("tag~='^{0}'", PubStreamPrefixer.prefix("")));
-         return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
+         return new FilterStream(new PubSubServerStateStream(), "tag~='DS_'");
+         //return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
       }
 
-      private void mySetText ()
-      {
-         this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty (myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
+      private void mySetText() {
+         this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
          myToolStripStatusLabelTime.Text = "Ready";
          myToolStripStatusLabelRows.Text = string.Empty;
       }
@@ -78,41 +88,36 @@ namespace BCG
       /// Gets the data in the form of a multilevel IDictionary
       /// </summary>
       /// <returns></returns>
-      private IDictionary myGetLibraryData ()
-      {
+      private IDictionary myGetLibraryData() {
          SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames ();
-         foreach (string pName in primitiveNames)
-         {
-            string aName = PrimitiveManager.instance.getAssemblyName (pName);
-            ICollection cNames = PrimitiveManager.instance.getConstructorSigns (pName);
-            if (!retVal.Contains (aName))
-               retVal.Add (aName, new SortedList ());
-            (retVal[aName] as IDictionary)[pName] = new ArrayList (cNames);
-            
+         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
+         foreach (string pName in primitiveNames) {
+            string aName = PrimitiveManager.instance.getAssemblyName(pName);
+            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
+            if (!retVal.Contains(aName))
+               retVal.Add(aName, new SortedList());
+            (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
+
          }
          foreach (DictionaryEntry outerDe in retVal)
             foreach (DictionaryEntry innerDe in (outerDe.Value as IDictionary))
-               (innerDe.Value as ArrayList).Sort ();
+               (innerDe.Value as ArrayList).Sort();
          return retVal;
       }
 
-      
-      private void myLoadAssemblies ()
-      {
-         myTreeView.Nodes.Clear ();
-         IDictionary libraryData = myGetLibraryData ();
-         foreach (DictionaryEntry de in libraryData)
-         {
-            TreeNode assemblyNode = new TreeNode (de.Key as String);
-            foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
-            { 
-               TreeNode primitiveNode = new TreeNode (innerDe.Key as string);
+
+      private void myLoadAssemblies() {
+         myTreeView.Nodes.Clear();
+         IDictionary libraryData = myGetLibraryData();
+         foreach (DictionaryEntry de in libraryData) {
+            TreeNode assemblyNode = new TreeNode(de.Key as String);
+            foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
+               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
                foreach (string con in (innerDe.Value as ICollection))
-                  primitiveNode.Nodes.Add (con);
-               assemblyNode.Nodes.Add (primitiveNode);
+                  primitiveNode.Nodes.Add(con);
+               assemblyNode.Nodes.Add(primitiveNode);
             }
-            myTreeView.Nodes.Add (assemblyNode);
+            myTreeView.Nodes.Add(assemblyNode);
          }
          /*
          myPrimitivesTextBox.Clear ();
@@ -126,27 +131,23 @@ namespace BCG
          }*/
       }
 
-      private void myHandleTableListChange (object o)
-      {
+      private void myHandleTableListChange(object o) {
          IBlueRowCollection rows = (o as DataPacket).newRows;
          myUpdateTableList();
       }
 
-      
+
       #region Persistence Keys
-      public int textBoxHeight
-      {
+      public int textBoxHeight {
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
       #endregion
-   
-      private void myTryExpression (string expStr)
-      {
-         myFreezeView (false); // force unfreeze the view
+
+      private void myTryExpression(string expStr) {
+         myFreezeView(false); // force unfreeze the view
          DataStreamExpression oldExp = myExp;
-         try 
-         {
+         try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
 
             if (exp == null)
@@ -156,7 +157,7 @@ namespace BCG
             myTabControl.SelectedIndex = 0;
             myErrorTextBox.Text = "";
             // myDataGrid.CaptionText = "";
-            
+
             if (myCurrentStream != null)
                myCurrentStream.onChange -= new BCG.DataStream.ChangeDelegate(myCallback);
             myCurrentStream = null;
@@ -164,7 +165,7 @@ namespace BCG
             myExp = exp;
             myCrono = new Crono();
             myCrono.start();
-            
+
             (myCurrentStream = myExp.getView(new Dictionary<string, StatementExpression>(0))).onChange += new DataStream.ChangeDelegate(myCallback);
             myChangeStyle = true;
             //new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback);
@@ -173,66 +174,55 @@ namespace BCG
                oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
                */
             //myQueryTextBox.Text = expStr;
-         }
-         catch (Exception exc)
-         {
+         } catch (Exception exc) {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
             myTabControl.SelectedIndex = 2;
-            myExp = oldExp;            
+            myExp = oldExp;
             myErrorTextBox.Text = exc.Message + "\n";
-            if(exc.InnerException != null)
+            if (exc.InnerException != null)
                myErrorTextBox.Text += "Inner Exception: " + exc.InnerException.Message + "\n";
             myErrorTextBox.Text += "\n" + exc.StackTrace;
             myCSharpText.Text = "";
          }
       }
 
-      private string myExtractFromClass (TextReader r)
-      {
+      private string myExtractFromClass(TextReader r) {
          string line = r.ReadLine();
-         while (line != null && line.IndexOf("*Expression*") == -1) 
+         while (line != null && line.IndexOf("*Expression*") == -1)
             line = r.ReadLine();
 
          if (line == null)
-            throw new Exception ("Unexpected EOF");
+            throw new Exception("Unexpected EOF");
 
          r.ReadLine();
 
          string result = "";
          line = r.ReadLine();
-         
-         while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1)
-         {
+
+         while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
             if (line != "")
                result += line.Substring("       ".Length) + "\n";
             line = r.ReadLine();
          }
          if (line == null)
-            throw new Exception ("Unexpected EOF");
+            throw new Exception("Unexpected EOF");
 
          return result;
       }
 
-      public void loadFile (string fileName)
-      {
-         try 
-         {
+      public void loadFile(string fileName) {
+         try {
             TextReader r = new StreamReader(fileName);
-            try 
-            {
+            try {
                myQueryTextBox.Text = myExtractFromClass(r);
-            }
-            finally
-            {
+            } finally {
                r.Close();
             }
             myLastFileName = fileName;
             this.Text = fileName;
-         }
-         catch (Exception exc)
-         {
+         } catch (Exception exc) {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
@@ -243,30 +233,28 @@ namespace BCG
 
       }
 
-      private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code)
-      {
+      private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
 
          myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
-         
-         myCSharpText.Text +="\n/*";
-         myCSharpText.Text +="\n   ----------------------------------------------------------------------";
-         myCSharpText.Text +="\n                               *Expression*";
-         myCSharpText.Text +="\n   ----------------------------------------------------------------------";
+
+         myCSharpText.Text += "\n/*";
+         myCSharpText.Text += "\n   ----------------------------------------------------------------------";
+         myCSharpText.Text += "\n                               *Expression*";
+         myCSharpText.Text += "\n   ----------------------------------------------------------------------";
          foreach (string line in myQueryTextBox.Text.Split('\n'))
-            myCSharpText.Text +="\n       " + line;
-         myCSharpText.Text +="\n\n   ----------------------------------------------------------------------";
-         myCSharpText.Text +="\n                                 *Schema*";
-         myCSharpText.Text +="\n   ----------------------------------------------------------------------";
-         foreach(ColumnInfo ci in schema.columnInfo) 
-            myCSharpText.Text +="\n       " + ci.colName + new string(' ', Math.Max(32 - ci.colName.Length,1) ) + ci.colType.ToString();
-         myCSharpText.Text +="\n*/\n";
-         
+            myCSharpText.Text += "\n       " + line;
+         myCSharpText.Text += "\n\n   ----------------------------------------------------------------------";
+         myCSharpText.Text += "\n                                 *Schema*";
+         myCSharpText.Text += "\n   ----------------------------------------------------------------------";
+         foreach (ColumnInfo ci in schema.columnInfo)
+            myCSharpText.Text += "\n       " + ci.colName + new string(' ', Math.Max(32 - ci.colName.Length, 1)) + ci.colType.ToString();
+         myCSharpText.Text += "\n*/\n";
+
 
          string lastVar = null;
          string variableDeclarations = "";
-         foreach (string var in vars) 
-         {
-            variableDeclarations+= "         ViewStream " + var + ";\n";
+         foreach (string var in vars) {
+            variableDeclarations += "         ViewStream " + var + ";\n";
             lastVar = var;
          }
 
@@ -278,18 +266,18 @@ namespace BCG
          else
             myLastClassName = "MyView";
 
-         myCSharpText.Text +="\n\n   public class " + myLastClassName + " : ViewStream \n   {\n";
-         
+         myCSharpText.Text += "\n\n   public class " + myLastClassName + " : ViewStream \n   {\n";
+
          myCSharpText.Text += "      public " + myLastClassName + "() : this(null) {}\n";
          myCSharpText.Text += "      public " + myLastClassName + "(ViewStreamDatabase db)\n      {\n";
 
          myCSharpText.Text += variableDeclarations;
-         
+
          if (lastVar != null)
             myCSharpText.Text += "\n";
          myCSharpText.Text += code + "\n";
          if (lastVar != null)
-            myCSharpText.Text += "         mySetRealSource(" + lastVar  + ");\n";
+            myCSharpText.Text += "         mySetRealSource(" + lastVar + ");\n";
          myCSharpText.Text += "      }\n";
          myCSharpText.Text += "   }\n";
          myCSharpText.Text += "}";
@@ -310,27 +298,31 @@ namespace BCG
             return Math.Max(header_width, min_width);
          }
       }
-
-      private void myCallback(object o)
-      {
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();
+      // this is the stream callback
+      private void myCallback(object o) {
+         _timer.start();
          DataPacket dp = o as DataPacket;
          myCrono.stop();
          Log.Ui.diag($"Received callback for {dp.schema.tableName}");
-         if (o == null)
-         {
-            System.Windows.Forms.MessageBox.Show (this, "Received a Null Data packet", 
+         if (o == null) {
+            System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
                                                    "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
             return;
          }
          Log.Ui.diag(dp.toIncrementalIDString());
-         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString ("hh:mm:ss:fff tt");
+         myToolStripStatusLabelTime.BackColor = Color.Black;
+         myToolStripStatusLabelRows.BackColor = Color.Black;
+         myToolStripStatusLabelTime.ForeColor = Color.White;
+         myToolStripStatusLabelRows.ForeColor = Color.White;
+         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
          myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + "  Added = " + dp.addedRows.Count + "  Deleted = " + dp.deletedRows.Count;
          Log.Ui.diag(Text);
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
 
-         myDataGrid.DataSource = new SimpleBindableCollection (dp.newRows.schema, dp.newRows);
-         if (myChangeStyle)
-         {
+         myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
+         var force = false;
+         if (myChangeStyle && force) {
             myChangeStyle = false;
             myDataGrid.Columns.Clear();
             myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
@@ -338,28 +330,30 @@ namespace BCG
          myCrono.start();
          // myDataGrid.CaptionText = myCrono.time.TotalMilliseconds + " ms ";
          // myDataGrid.CaptionText += dp.newRows.Count.ToString () + " records";
-       
+
 
          myDataGrid.HorizScrollBarValue = hscrollval;
 
          var varNames = new List<string>(0);
-         string code = myExp.print(9,varNames);
-         myPrintCode(dp.schema ,varNames, code);
+         string code = myExp.print(9, varNames);
+         myPrintCode(dp.schema, varNames, code);
+         myConsoleRichTextBox.Clear();
+         // this was was very very slow
+         //myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30);
+         _timer.stop();
+         var msecs = _timer.lastIntervalTime;
+         bts.Log.Gen.info($"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {msecs}");
       }
 
-      private Crono  myCrono;
- 
+      private Crono myCrono;
+
       private DataStreamExpression myExp;
 
-      private void myTimer_Tick(object sender, System.EventArgs e)
-      {
-         try 
-         {
-            ViewStreamDatabase.instance.doWork ();
+      private void myTimer_Tick(object sender, System.EventArgs e) {
+         try {
+            ViewStreamDatabase.instance.doWork();
             MasterWorker.getInstance(false).doWork();
-         }
-         catch (Exception exc)
-         {
+         } catch (Exception exc) {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
@@ -370,27 +364,22 @@ namespace BCG
          }
       }
 
-      class StringWrapper
-      {
-         public StringWrapper(string s)
-         {
+      class StringWrapper {
+         public StringWrapper(string s) {
             myTableName = s;
          }
-         public string tableName 
-         {
-            get{return myTableName;}
+         public string tableName {
+            get { return myTableName; }
          }
          private string myTableName;
       }
 
-      private void MainForm_Closed(object sender, System.EventArgs e)
-      {
+      private void MainForm_Closed(object sender, System.EventArgs e) {
          if (myExp != null)
             myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
       }
 
-      private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
          myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionColor = Color.Black;
@@ -398,49 +387,44 @@ namespace BCG
          myTryExpression(myQueryTextBox.Text);
       }
 
-      private void myToolStripMenuItemExecSel_Click (object sender, EventArgs e)
-      {
-         int selstart = myQueryTextBox.SelectionStart;
-         int sellength = myQueryTextBox.SelectionLength;
-         myQueryTextBox.SelectionStart = 0;
-         myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
-         myQueryTextBox.SelectionColor = Color.Black;
-         myQueryTextBox.SelectionStart = selstart;
-         myQueryTextBox.SelectionLength = sellength;
-         myQueryTextBox.SelectionColor = Color.Crimson;
-         myTryExpression (myQueryTextBox.SelectedText);
-         myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
-         myQueryTextBox.SelectionLength = 0;
-         myQueryTextBox.SelectionColor = Color.Black;
+      private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
+         // int selstart = myQueryTextBox.SelectionStart;
+         // int sellength = myQueryTextBox.SelectionLength;
+         // myQueryTextBox.SelectionStart = 0;
+         // myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
+         // myQueryTextBox.SelectionColor = Color.White;
+         // myQueryTextBox.SelectionStart = selstart;
+         // myQueryTextBox.SelectionLength = sellength;
+         // myQueryTextBox.SelectionColor = Color.Crimson;
+         // myTryExpression(myQueryTextBox.SelectedText);
+         // myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
+         // myQueryTextBox.SelectionLength = 0;
+         // myQueryTextBox.SelectionColor = Color.White;
       }
 
-      private void myMenuItemOpenClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemOpenClick(object sender, System.EventArgs e) {
          OpenFileDialog od = new OpenFileDialog();
          od.Filter = "C# class (*.cs)|*.cs";
 
-         if(od.ShowDialog() == DialogResult.OK)
+         if (od.ShowDialog() == DialogResult.OK)
             loadFile(od.FileName);
       }
 
-      private void mySave(string fileName)
-      {
+      private void mySave(string fileName) {
          using (TextWriter w = new StreamWriter(fileName))
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
       }
 
-      private void myMenuItemSaveClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemSaveClick(object sender, System.EventArgs e) {
          if (myLastFileName == null)
-            myMenuItemSaveAsClick(sender,e);
+            myMenuItemSaveAsClick(sender, e);
          else
             mySave(myLastFileName);
       }
 
-      private void myMenuItemSaveAsClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
          SaveFileDialog sd = new SaveFileDialog();
          if (myLastFileName != null)
             sd.FileName = myLastFileName;
@@ -448,62 +432,53 @@ namespace BCG
             sd.FileName = myLastClassName + ".cs";
 
          if (sd.ShowDialog() == DialogResult.OK)
-            mySave (sd.FileName);
+            mySave(sd.FileName);
       }
 
-      private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
-      {
-         if (myTableListBox.SelectedItem != null) 
-         {
-            string tableName = myTableListBox.SelectedItem.ToString ();
+      private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
+         if (myTableListBox.SelectedItem != null) {
+            string tableName = myTableListBox.SelectedItem.ToString();
             myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
-            myTryExpression (myQueryTextBox.Text);
+            myTryExpression(myQueryTextBox.Text);
          }
       }
 
-      private void myDataGrid_MouseEnter(object sender, System.EventArgs e)
-      {
+      private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
          //myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
          // if (myDataGrid.TableStyles.Count > 0)
          //    myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
       }
 
-      private void myDataGrid_MouseLeave(object sender, System.EventArgs e)
-      {
+      private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
          //myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
          // if (myDataGrid.TableStyles.Count > 0)
          //    myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
       }
 
-      private void myCSharpText_MouseEnter(object sender, System.EventArgs e)
-      {
-         myCSharpText.BackColor = Color.Gainsboro;
-         myCSharpText.Font = new Font (myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
+      private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
+         //myCSharpText.BackColor = Color.Gainsboro;
+         myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
       }
 
-      private void myCSharpText_MouseLeave(object sender, System.EventArgs e)
-      {
-         myCSharpText.BackColor = Color.White;
-         myCSharpText.Font = new Font (myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
+      private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
+         //myCSharpText.BackColor = Color.White;
+         myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
       }
 
-      private void myTableListBox_MouseEnter(object sender, System.EventArgs e)
-      {
-         myTableListBox.BackColor = Color.Gainsboro;
-         myTableListBox.Font = new Font (myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
+      private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
+         //myTableListBox.BackColor = Color.Gainsboro;
+         myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
       }
 
-      private void myTableListBox_MouseLeave(object sender, System.EventArgs e)
-      {
-         myTableListBox.BackColor = Color.White;
-         myTableListBox.Font = new Font (myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
+      private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
+         //myTableListBox.BackColor = Color.White;
+         myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
       }
 
-      private void myLoad (object sender, EventArgs e)
-      {         
-         myFreezeView (false);
+      private void myLoad(object sender, EventArgs e) {
+         myFreezeView(false);
 
-         myToolStripMenuItemSite.DropDownItems.Clear ();
+         myToolStripMenuItemSite.DropDownItems.Clear();
          // foreach (var name in SitesCfg.instance.siteNames)
          // {
          //    ToolStripMenuItem mi = new ToolStripMenuItem (name);
@@ -517,144 +492,123 @@ namespace BCG
          // }
 
          myConsoleRichTextBox.ReadOnly = true;
-         myConsoleOutput = new RichTextBoxStreamWriter (myConsoleRichTextBox);
-         Console.SetOut (myConsoleOutput);
-         Console.SetError (myConsoleOutput);
-         if(BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-11, myConsoleRichTextBox.Handle) == 0) // set stdout
+         myConsoleOutput = new RichTextBoxStreamWriter(myConsoleRichTextBox);
+         Console.SetOut(myConsoleOutput);
+         Console.SetError(myConsoleOutput);
+         if (BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-11, myConsoleRichTextBox.Handle) == 0) // set stdout
          {
-            var ex = new Win32Exception ();  // same as new Win32Exception(Marshal.GetLastWin32Error())
-            Console.WriteLine (ex.Message);
-         }
-         else
-         {
-            Console.WriteLine ("Set stdout worked.");
+            var ex = new Win32Exception();  // same as new Win32Exception(Marshal.GetLastWin32Error())
+            Console.WriteLine(ex.Message);
+         } else {
+            Console.WriteLine("Set stdout worked.");
          }
 
-         if(BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-12, myConsoleRichTextBox.Handle) == 0) // set stderr
+         if (BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-12, myConsoleRichTextBox.Handle) == 0) // set stderr
          {
-            var ex = new Win32Exception ();  // same as new Win32Exception(Marshal.GetLastWin32Error())
-            Console.WriteLine (ex.Message);
+            var ex = new Win32Exception();  // same as new Win32Exception(Marshal.GetLastWin32Error())
+            Console.WriteLine(ex.Message);
+         } else {
+            Console.WriteLine("Set stderr worked.");
          }
-         else
-         {
-            Console.WriteLine ("Set stderr worked.");
-         }
-         Console.WriteLine ("Redirecting console ouput to this window...");
+         Console.WriteLine("Redirecting console ouput to this window...");
 
-         myLoadAssemblies ();
+         myLoadAssemblies();
 
-         ViewStreamDatabase.instance = new ViewStreamDatabase ();
-         myAllTagsStream = myGetAllTagsStream ();
-         myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate (myHandleTableListChange);
+         ViewStreamDatabase.instance = new ViewStreamDatabase();
+         myAllTagsStream = myGetAllTagsStream();
+         myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
          myTabControl.SelectedIndex = 3;
-         myTreeView.AfterSelect += new TreeViewEventHandler (myHandleTreeNodeSelect);
+         myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
          //do this lazily so we don't have to have bps, etc up to run dsqa
          //_library = new LibraryForm(this);
       }
 
-      private void myMenuItemSiteClick (object sender, System.EventArgs e)
-      {
-         ToolStripMenuItem mi = (ToolStripMenuItem) sender;
+      private void myMenuItemSiteClick(object sender, System.EventArgs e) {
+         ToolStripMenuItem mi = (ToolStripMenuItem)sender;
 
-         if (mi.Text != mySite)
-         {
+         if (mi.Text != mySite) {
             mySite = mi.Text;
             foreach (var m in mySiteItems)
                m.Checked = false;
             mi.Checked = true;
 
-            myAllTagsStream.onChange -= new BCG.DataStream.ChangeDelegate (myHandleTableListChange);
-            myTableListBox.Items.Clear ();
+            myAllTagsStream.onChange -= new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
+            myTableListBox.Items.Clear();
 
             // SitesCore.setMySiteName (mySite);
-            
-            ViewStreamDatabase.instance = new ViewStreamDatabase ();
-            myAllTagsStream = myGetAllTagsStream ();
-            myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate (myHandleTableListChange);
+
+            ViewStreamDatabase.instance = new ViewStreamDatabase();
+            myAllTagsStream = myGetAllTagsStream();
+            myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
 
             myDataGrid.DataSource = null;
             myTabControl.SelectedIndex = 3;
-            mySetText ();
+            mySetText();
          }
       }
 
-      private void myClosing (object sender, CancelEventArgs e)
-      {
+      private void myClosing(object sender, CancelEventArgs e) {
       }
 
-      private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e)
-      {
-         if (myTreeView.SelectedNode != null)
-         {
-            if (myTreeView.SelectedNode.LastNode == null)
-            {
+      private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
+         if (myTreeView.SelectedNode != null) {
+            if (myTreeView.SelectedNode.LastNode == null) {
                string sn = myTreeView.SelectedNode.Parent.Text + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
-      private void myToolStripMenuItemExit_Click (object sender, EventArgs e)
-      {
-         Close ();
+      private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
+         Close();
       }
 
-      private void myToolStripMenuItemFreeze_Click (object sender, EventArgs e)
-      {
-         myFreezeView (myTimer.Enabled);
+      private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
+         myFreezeView(myTimer.Enabled);
       }
 
-      private void myFreezeView (bool freeze)
-      {
+      private void myFreezeView(bool freeze) {
          myTimer.Enabled = !freeze;
          myToolStripMenuItemFreeze.Checked = freeze;
          myToolStripMenuItemStep.Enabled = freeze;
       }
 
-      private void myToolStripMenuItemStep_Click (object sender, EventArgs e)
-      {
-         ViewStreamDatabase.instance.doWork ();
+      private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
+         ViewStreamDatabase.instance.doWork();
       }
 
-      private void myToolStripMenuItemLoadAss_Click (object sender, EventArgs e)
-      {
-         OpenFileDialog ofd = new OpenFileDialog ();
+      private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
+         OpenFileDialog ofd = new OpenFileDialog();
          ofd.ShowReadOnly = true;
-         DialogResult dr = ofd.ShowDialog (this);
-         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
-         {
-            PrimitiveManager.instance.loadAssembly (ofd.FileName);
-            myLoadAssemblies ();
+         DialogResult dr = ofd.ShowDialog(this);
+         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
+            PrimitiveManager.instance.loadAssembly(ofd.FileName);
+            myLoadAssemblies();
          }
       }
 
-      private void myToolStripMenuItemTest_Click (object sender, EventArgs e)
-      {
-         if (myCurrentStream != null)
-         {
-            myCurrentStream.onChange -= new DataStream.ChangeDelegate (myCallback);
-            myCurrentStream.onChange += new DataStream.ChangeDelegate (myCallback);
+      private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
+         if (myCurrentStream != null) {
+            myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
+            myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
          }
       }
 
-      private void myToolStripMenuItemClearConsole_Click (object sender, EventArgs e)
-      {
-         myConsoleRichTextBox.Clear ();
-         Console.WriteLine ("Redirecting console ouput to this window...");
+      private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
+         myConsoleRichTextBox.Clear();
+         Console.WriteLine("Redirecting console ouput to this window...");
       }
 
-      private void myTableFilterBox_TextChanged(object sender, EventArgs e)
-      {                     
+      private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
          myUpdateTableList();
       }
 
-      private void myUpdateTableList()
-      {
+      private void myUpdateTableList() {
          IBlueRowCollection rows = myAllTagsStream.rows;
          if (rows == null)
             return;
-         
-      
+
+
          ICollection oldItems = myTableListBox.Items;
 
          myTableListBox.BeginUpdate();
@@ -664,15 +618,12 @@ namespace BCG
          myTableListBox.Items.Clear();
          SortedList<string, bool> sl = new SortedList<string, bool>();
          if (string.IsNullOrEmpty(myTableFilterBox.Text))
-            foreach (IBlueRow r in rows)
-            {
+            foreach (IBlueRow r in rows) {
                sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true;
             }
-         else
-         {
+         else {
             Regex tableRegex = new Regex(myTableFilterBox.Text);
-            foreach (IBlueRow r in rows)
-            {
+            foreach (IBlueRow r in rows) {
                string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length);
                if (tableRegex.IsMatch(thisTable))
                   sl[thisTable] = true;
@@ -687,11 +638,10 @@ namespace BCG
             myTableListBox.TopIndex = top;
          else
             myTableListBox.TopIndex = Math.Min(sl.Count - 1, topindex);
-         myTableListBox.EndUpdate();         
+         myTableListBox.EndUpdate();
       }
 
-      private void testGridToolStripMenuItem_Click(object sender, EventArgs e)
-      {
+      private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
          HighPerformanceTimer timer = new HighPerformanceTimer();
          //while (true)
          //{
@@ -713,11 +663,9 @@ namespace BCG
          foreach (IBlueRow r in s.rows)
             rows.Add(r);
 
-         for (int i = 0; i < 100; i++)
-         {
+         for (int i = 0; i < 100; i++) {
             timer.start();
-            rows.Sort(delegate(IBlueRow r1, IBlueRow r2)
-            {
+            rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
                return _sort_dicts_index(r1, r2, 5, true);
             });
             timer.stop();
@@ -725,22 +673,19 @@ namespace BCG
          }
       }
 
-      private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high)
-      {
+      private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
          if (r1.Contains(column) && r2.Contains(column))
             return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
          else
             return 0;
       }
 
-      private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high)
-      {
+      private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
          //return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
          return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
       }
 
-      internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high)
-      {
+      internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
          if (myIsDontCare(o1))
             return myIsDontCare(o2) ? 0 : 1;
          else
@@ -748,16 +693,14 @@ namespace BCG
                                                     : ((IComparable)o2).CompareTo(o1));
       }
 
-      private static bool myIsDontCare(object o)
-      {
+      private static bool myIsDontCare(object o) {
          return o == null || o == DBNull.Value ||
                   o is string && String.IsNullOrEmpty((string)o) ||
                   o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
                   o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
       }
 
-      private void _new_grid_form(object args)
-      {
+      private void _new_grid_form(object args) {
          //System.Timers.Timer t = new System.Timers.Timer();
          ////t.SynchronizingObject
          //BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
@@ -768,5 +711,13 @@ namespace BCG
       }
 
       private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
+
+      private void myTableFilterLabel_Click(object sender, EventArgs e) {
+
+      }
+
+      private void MainForm_Load(object sender, EventArgs e) {
+
+      }
    }
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.resx

@@ -1,64 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!-- 
-    Microsoft ResX Schema 
-    
-    Version 2.0
-    
-    The primary goals of this format is to allow a simple XML format 
-    that is mostly human readable. The generation and parsing of the 
-    various data types are done through the TypeConverter classes 
-    associated with the data types.
-    
-    Example:
-    
-    ... ado.net/XML headers & schema ...
-    <resheader name="resmimetype">text/microsoft-resx</resheader>
-    <resheader name="version">2.0</resheader>
-    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
-    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
-    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
-    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
-    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
-        <value>[base64 mime encoded serialized .NET Framework object]</value>
-    </data>
-    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
-        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
-        <comment>This is a comment</comment>
-    </data>
-                
-    There are any number of "resheader" rows that contain simple 
-    name/value pairs.
-    
-    Each data row contains a name, and value. The row also contains a 
-    type or mimetype. Type corresponds to a .NET class that support 
-    text/value conversion through the TypeConverter architecture. 
-    Classes that don't support this are serialized and stored with the 
-    mimetype set.
-    
-    The mimetype is used for serialized objects, and tells the 
-    ResXResourceReader how to depersist the object. This is currently not 
-    extensible. For a given mimetype the value must be set accordingly:
-    
-    Note - application/x-microsoft.net.object.binary.base64 is the format 
-    that the ResXResourceWriter will generate, however the reader can 
-    read any of the formats listed below.
-    
-    mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with 
-            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
-            : and then encoded with base64 encoding.
-    
-    mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with 
-            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
-            : and then encoded with base64 encoding.
-
-    mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array 
-            : using a System.ComponentModel.TypeConverter
-            : and then encoded with base64 encoding.
-    -->
   <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
     <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
     <xsd:element name="root" msdata:IsDataSet="true">

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -1,32 +1,40 @@
-using System;
-using System.Windows.Forms;
 using BCG;
-using System.IO;
-using System.Runtime.Serialization;
-using System.Runtime.Serialization.Formatters.Binary;
-using bts;
-using mbts;
-using mbts.config;
+
 using bts.utils;
 
-static class Program
-{
-   [STAThread]
-   static void Main(string[] args)
-   {
+using mbts.config;
+
+using System;
+using System.Linq;
+using System.Windows.Forms;
+using System.Threading.Tasks;
+using System.ComponentModel;
+
+static class Program {
+   static void init(string[] args) {
+      Func<string, string, Task> login = async (l, p) => {
+         var login = l;
+         var passwd = p;
+         var service_ctx = bts.api.core.ServiceContext.create();
+         await service_ctx.login(login, passwd);
+      };
       BTSNativeMethods.provide_command_line(args, args.Length);
       Configuration.initialize();
-      bts.Log.Gen.info($"Starting DSQA");
-      Console.WriteLine("Starting DataStreamQueryAnalyzer...");
-      Console.WriteLine("Starting Test Publisher");
-      // TestPubliser.publish();
-      // TestPubliser.subscribe();
+      Task.Run(async () => await login("bts", String.Empty));
+   }
+   [STAThread]
+   static void Main(string[] args) {
+      init(args);
+      string args_concat = String.Empty;
+      args.ToList().ForEach(t => args_concat += t);
+      var init_log = $"Starting DSQA args: {args_concat} len: {args.Length}";
+      bts.Log.Gen.info(init_log);
+      Console.WriteLine(init_log);
       Console.WriteLine("Launching Main Window");
       MainForm mainForm = new MainForm();
-      if (args.Length > 0)
-      {
+      if (args.Length > 0) {
          mainForm.loadFile(args[0]);
       }
       Application.Run(mainForm);
    }
-}
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -1,4 +1,3 @@
-
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31903.59
@@ -21,8 +20,24 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_core", "..\client\spa
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_api", "..\client\spark_api\spark_api.csproj", "{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlueRegistry", "BlueRegistry\BlueRegistry.csproj", "{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "BaseUtilities\BaseUtilities.csproj", "{A9EDEEBD-A03A-4281-8FAF-2419D3E7009C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GuiUtilities", "GuiUtilities\GuiUtilities.csproj", "{CADBA1B3-0837-44DD-85BB-70351AD3A028}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "ArithmeticExpressions\ArithmeticExpressions.csproj", "{EFE6AD17-2564-43A1-B0E5-A2A8DA173002}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "DataStreamExpressions\DataStreamExpressions.csproj", "{D46A152D-AD29-4BEF-980B-5896694E312F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "ColumnMapExpressions\ColumnMapExpressions.csproj", "{A844AB94-195B-4A7F-89CD-9A335985328E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "ExpressionCore\ExpressionCore.csproj", "{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStreamTests\DataStreamTests.csproj", "{66AB10D3-155F-4663-B1BC-D0818E17B87B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -100,5 +115,9 @@ Global
 		{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -9,8 +9,7 @@ using System.IO;
 using System.Threading;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
    /// from the server, and in case of missing hbs, informs its clients, and tries to
@@ -23,8 +22,7 @@ namespace BCG
    /// </summary>
    public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
                                                    ITCPMessageHandler,
-                                                   ITCPConnectionHandler
-   {
+                                                   ITCPConnectionHandler {
       private ITCPConnection _connection, _new_connection;
       private readonly BlutoSubAddress _server_address;
       // if a message(with confirm) can not be sent , its id is queued here, 
@@ -43,98 +41,75 @@ namespace BCG
       public PubSubServerManagedTCPConnection(string serverURL)
          : this(new BlutoSubAddress(serverURL)) { }
 
-      public PubSubServerManagedTCPConnection(BlutoSubAddress server_address)
-      {
+      public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
          _server_address = server_address;
          _connect_to_ps_server();
       }
 
-      private void _connect_to_ps_server()
-      {
+      private void _connect_to_ps_server() {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
          ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
       }
 
       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id)
-      {
+      public void sendWithConfirmation(byte[] data, ref string conf_id) {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
-         if (_connection != null)
-         {
-            _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) =>
-            {
+         if (_connection != null) {
+            _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
                if (_on_send_onfirm_callback != null)
                   _on_send_onfirm_callback(msg_id, confirmed);
             });
-         }
-         else
-         {
+         } else {
             bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
             conf_id = Guid.NewGuid().ToString();
             _negative_unsent_confs.enqueue(conf_id);
          }
       }
 
-      public void send(byte[] data)
-      {
+      public void send(byte[] data) {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
-         if (_connection != null)
-         {
+         if (_connection != null) {
             bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
             _connection.send(data);
          }
       }
 
-      public void send(Stream str)
-      {
+      public void send(Stream str) {
          BinaryReader br = new BinaryReader(str);
          send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
       }
 
-      public void send(PubSubMessage msg)
-      {
+      public void send(PubSubMessage msg) {
          send(msg.to_stream());
       }
 
-      public void send_with_confirmation(PubSubMessage msg, ref string conf_id)
-      {
+      public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
          var bytes = msg.to_bytes();
          sendWithConfirmation(bytes, ref conf_id);
       }
 
-      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
-      {
+      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
          return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
       }
 
-      public string serverURL
-      {
+      public string serverURL {
          get { return _server_address.ToString(); }
       }
 
-      public string clientURL
-      {
-         get
-         {
+      public string clientURL {
+         get {
             string rv = "";
-            lock (this)
-            {
-               if (_connection == null)
-               {
+            lock (this) {
+               if (_connection == null) {
                   rv = "";
-               }
-               else
-               {
-                  if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty)
-                  {
+               } else {
+                  if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) {
                      string ip;
                      int port;
                      _connection.getSelfAddress(out ip, out port);
                      rv = new BlutoSubAddress(ip, port).ToString();
-                  }
-                  else
-                  {
+                  } else {
                      rv = "";
                   }
                }
@@ -143,18 +118,15 @@ namespace BCG
          }
       }
 
-      public void registerCallback(ConnectionStatusChange del)
-      {
+      public void registerCallback(ConnectionStatusChange del) {
          _conn_callback = del;
       }
 
-      public void registerDataCallback(DataDelegate del)
-      {
+      public void registerDataCallback(DataDelegate del) {
          _data_callback = del;
       }
 
-      public void registerSendConfirmationCallback(SendConfirmationDelegate del)
-      {
+      public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
          _on_send_onfirm_callback = del;
       }
 
@@ -163,21 +135,17 @@ namespace BCG
       #region IDisposable Members
 
       private bool _is_disposed = false;
-      public void Dispose()
-      {
+      public void Dispose() {
          bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
          //throw new Exception("The method or operation is not implemented.");
-         if (_connection != null)
-         {
+         if (_connection != null) {
             _connection.Dispose();
 
          }
          _is_disposed = true;
       }
-      public bool isDisposed
-      {
-         get
-         {
+      public bool isDisposed {
+         get {
             return _is_disposed;
          }
 
@@ -193,12 +161,10 @@ namespace BCG
       /// (long) is safe for multi threaded operations
       /// </summary>
       /// <param name="message"></param>
-      public void onData(RemoteMessage message)
-      {
+      public void onData(RemoteMessage message) {
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
          // TODO check for disconenction
-         switch (message.messageType)
-         {
+         switch (message.messageType) {
             case RemoteMessageType.Data:
                if (_data_callback != null)
                   _data_callback(message.remoteConnectionInfo.ToString(), message.message);
@@ -219,17 +185,18 @@ namespace BCG
                break;
 
          }
-         if (message.messageType == RemoteMessageType.Data)
-         {
-            if (_data_callback != null)
-               _data_callback(message.remoteConnectionInfo.ToString(), message.message);
-            else
-               _in_data_queue.enqueue(message.message);
-         }
-         else
-         {
-            _last_hb = DateTime.Now;
-         }
+         // this stupid shit, caused duplicating og messages and 2 days of wild goose chase
+         // if (message.messageType == RemoteMessageType.Data)
+         // {
+         //    if (_data_callback != null)
+         //       _data_callback(message.remoteConnectionInfo.ToString(), message.message);
+         //    else
+         //       _in_data_queue.enqueue(message.message);
+         // }
+         // else
+         // {
+         //    _last_hb = DateTime.Now;
+         // }
       }
 
       #endregion
@@ -241,15 +208,13 @@ namespace BCG
       /// except onData)
       /// </summary>
       /// <param name="connection"></param>
-      public void onConnection(ITCPConnection connection)
-      {
+      public void onConnection(ITCPConnection connection) {
          string ip;
          int port;
          connection.getPeerAddress(out ip, out port);
          BlutoSubAddress add = new BlutoSubAddress(ip, port);
          bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
-         if (add.ToString() == _server_address.ToString())
-         {
+         if (add.ToString() == _server_address.ToString()) {
             _last_hb = DateTime.Now;
             _new_connection = connection;
          }
@@ -257,39 +222,31 @@ namespace BCG
 
       #endregion
 
-      void _dispose_and_reconnect_if_missing_hb()
-      {
-         if (!_connection.isDisposed)
-         {
+      void _dispose_and_reconnect_if_missing_hb() {
+         if (!_connection.isDisposed) {
             TimeSpan hb_interval = DateTime.Now - _last_hb;
             bool hb_missed = hb_interval > _connection_drop_threshold;
-            if (hb_missed)
-            {
+            if (hb_missed) {
                if (_conn_callback != null)
                   _conn_callback(_server_address.ToString(), false);
                bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
                _connection.Dispose();
                _connect_to_ps_server();
             }
-         }
-         else
-         {
+         } else {
             // already disposed, nothing to do
          }
       }
 
-      void _check_for_new_connection()
-      {
-         if (_new_connection != null)
-         {
+      void _check_for_new_connection() {
+         if (_new_connection != null) {
             _connection = _new_connection;
             _new_connection = null; // since we are using its presence as a marker of nw connections
             bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
             _connection.register(this);
             if (_conn_callback != null)
                _conn_callback(_server_address.ToString(), true);
-            if (_on_send_onfirm_callback != null)
-            {
+            if (_on_send_onfirm_callback != null) {
                var msg_ids = _negative_unsent_confs.dequeueAll();
                foreach (var id in msg_ids)
                   _on_send_onfirm_callback(id, false);
@@ -304,8 +261,7 @@ namespace BCG
       /// the design only allows for onConnection to be on a different
       /// thread.
       /// </summary>
-      public void doWork()
-      {
+      public void doWork() {
          //_dispose_and_reconnect_if_missing_hb();
          _check_for_new_connection();
          ///check for heartbeats and let clients know in 

src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs

@@ -3,10 +3,8 @@ using BCG;
 //using BCG.Utilities;
 using System.Text.RegularExpressions;
 
-namespace BCG
-{
-   public class PubSubRoute : IComparable<PubSubRoute>
-   {
+namespace BCG {
+   public class PubSubRoute : IComparable<PubSubRoute> {
       public string serverAddress;
       public string tagRegExp;
       //route names are used in intermediary code (i.e. while substations are being deployed)
@@ -16,8 +14,7 @@ namespace BCG
 
       public PubSubRoute() { }
 
-      public PubSubRoute(string theServerAddress, string theTagRegExp)
-      {
+      public PubSubRoute(string theServerAddress, string theTagRegExp) {
          serverAddress = theServerAddress;
          tagRegExp = theTagRegExp;
       }
@@ -28,41 +25,39 @@ namespace BCG
       //    // tagRegExp = cfg.tagsRegex;
       // }
 
-      public string[] getAllPubSubs()
-      {
+      public string[] getAllPubSubs() {
          return serverAddress.Split(';');
       }
 
-      public Regex getRegex()
-      {
+      public Regex getRegex() {
          return myRegex ?? new Regex(tagRegExp);
       }
 
       #region IComparable<PubSubRoute> Members
 
-      public int CompareTo(PubSubRoute other)
-      {
+      public int CompareTo(PubSubRoute other) {
          return serverAddress == other.serverAddress ? tagRegExp.CompareTo(other.tagRegExp)
                                                       : serverAddress.CompareTo(other.serverAddress);
       }
 
       #endregion
 
-      public override bool Equals(object obj)
-      {
+      public override bool Equals(object obj) {
          if (!(obj is PubSubRoute) || obj == null)
             return false;
-         else
-         {
+         else {
             PubSubRoute other = obj as PubSubRoute;
             return serverAddress.Equals(other.serverAddress) &&
                      tagRegExp.Equals(other.tagRegExp);
          }
       }
 
-      public override int GetHashCode()
-      {
+      public override int GetHashCode() {
          return base.GetHashCode();
       }
+
+      public static PubSubRoute get_default_route() {
+         return new PubSubRoute("bluto://127.0.0.0.1:1976", ".*");
+      }
    }
 }

src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs

@@ -7,26 +7,22 @@ using BCG;
 //using BCG.Utilities;
 //using bts;
 
-namespace BCG
-{
-   public class PubSubRouter : IPubSubRouter
-   {
+namespace BCG {
+   public class PubSubRouter : IPubSubRouter {
       private Regex[] myPubRouteRegExps;
       private Regex[] mySubRouteRegExps;
       private PubSubRoute[] myPubRoutes;
       private PubSubRoute[] mySubRoutes;
+      private readonly string _temp_only_pubsub = "bluto://127.0.0.1:1976";
 
 
       public PubSubRouter() : this(
          PubSub3Config.instance.getSubRouting(),
-         PubSub3Config.instance.getPubRouting())
-      { }
+         PubSub3Config.instance.getPubRouting()) { }
 
-      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes)
-      { }
+      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
 
-      public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes)
-      {
+      public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes) {
          mySubRouteRegExps = new Regex[subRoutes.Length];
          myPubRouteRegExps = new Regex[pubRoutes.Length];
 
@@ -39,8 +35,7 @@ namespace BCG
             myPubRouteRegExps[i] = new Regex(pubRoutes[i].tagRegExp);
       }
 
-      private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes)
-      {
+      private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes) {
          for (int i = 0; i < routes.Length; i++)
             if (regexes[i].IsMatch(forTag))
                return i;
@@ -50,49 +45,43 @@ namespace BCG
 
       #region IPubSubRouter methods
 
-      private string myPickAServer(string tag, string[] urls)
-      {
+      private string myPickAServer(string tag, string[] urls) {
          if (urls.Length == 1)
             return urls[0];
-         else
-         {
+         else {
             uint hc = (uint)tag.GetHashCode();
             return urls[hc % urls.Length];
          }
       }
 
-      public string getPubURLForTag(string tag)
-      {
-         var rv = "bluto://127.0.0.1:1976"; //myPickAServer(tag, myPubRoutes[myGetRouteIndex(tag, myPubRoutes, myPubRouteRegExps)].getAllPubSubs());
+      public string getPubURLForTag(string tag) {
+         var rv = _temp_only_pubsub;
          bts.Log.Gen.info($"PubSubRouter.getPubURLForTag tag: {tag} route: {rv}");
          return rv;
       }
 
-      public string getSubURLForTag(string tag)
-      {
-         var rv = "bluto://127.0.0.1:1976"; // myPickAServer(tag, mySubRoutes[myGetRouteIndex(tag, mySubRoutes, mySubRouteRegExps)].getAllPubSubs());
+      public string getSubURLForTag(string tag) {
+         var rv = _temp_only_pubsub; // myPickAServer(tag, mySubRoutes[myGetRouteIndex(tag, mySubRoutes, mySubRouteRegExps)].getAllPubSubs());
          bts.Log.Gen.info($"PubSubRouter.getSubURLForTag tag: {tag} route: {rv}");
          return rv;
       }
 
-      public List<string> getAllPubSubSubURLs()
-      {
-         List<string> theURLs = new List<string>();
-
-         foreach (PubSubRoute subRoute in mySubRoutes)
-            theURLs.AddRange(subRoute.getAllPubSubs());
-         return theURLs;
+      public List<string> getAllPubSubSubURLs() {
+         // List<string> theURLs = new List<string>();
 
+         // foreach (PubSubRoute subRoute in mySubRoutes)
+         //    theURLs.AddRange(subRoute.getAllPubSubs());
+         // return theURLs;
+         return new List<string>() { _temp_only_pubsub };
       }
 
-      public List<string> getAllPubSubPubURLs()
-      {
-         List<string> theURLs = new List<string>();
-
-         foreach (PubSubRoute pubRoute in myPubRoutes)
-            theURLs.AddRange(pubRoute.getAllPubSubs());
-         return theURLs;
+      public List<string> getAllPubSubPubURLs() {
+         // List<string> theURLs = new List<string>();
 
+         // foreach (PubSubRoute pubRoute in myPubRoutes)
+         //    theURLs.AddRange(pubRoute.getAllPubSubs());
+         // return theURLs;
+         return new List<string>() { _temp_only_pubsub };
       }
       public event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
       public event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
@@ -100,10 +89,8 @@ namespace BCG
       #endregion
 
       //this function is put here so that the events are used, in order to prevent build errors. 
-      private void myNeverCallThisFunction()
-      {
-         if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null)
-         {
+      private void myNeverCallThisFunction() {
+         if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) {
             int aVar = 2;
             aVar = aVar + 2;
          }

src/DataStreamSuite/PubSub/Client/TagTypeToAssembly.cs

@@ -1,4 +1,6 @@
 using System;
+using System.IO;
+using System.Reflection;
 
 namespace BCG {
    [Serializable]
@@ -6,8 +8,11 @@ namespace BCG {
       //[Serializable]
       public class Entry {
          public string handlerClass = "BCG.PubSubServerTagHandler";
-         // TODOds
-         public string assembly = "C:\\Users\\m_tah\\spark\\build\\Debug\\bin\\net7.0\\DataStreamCore.dll";
+
+         public string assembly = Path.Combine(
+            Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!,
+            "DataStreamCore.dll");
+
          public override string ToString() {
             return $"handlerClass: {handlerClass} assembly: {assembly}";
          }

src/DataStreamSuite/PubSub/Config/PubSub3Config.cs

@@ -169,24 +169,14 @@ namespace BCG {
 
       public PubSubRoute[] getRouting() {
          // TODO
-         return new PubSubRoute[] { };
+         return new PubSubRoute[] { PubSubRoute.get_default_route() };
          // return getRouting(SitesCore.getMySiteName());
       }
 
       public PubSubRoute[] getRouting(string forSite) {
          // TODO
-         return new PubSubRoute[] { };
-         // if (sites_layout != null) {
-         //    foreach (var s in sites_layout.sites)
-         //       if (s.name == forSite) {
-         //          List<PubSubRoute> result = new List<PubSubRoute>();
-         //          foreach (var cfgRoute in s.pubSub.routes)
-         //             result.Add(new PubSubRoute(cfgRoute));
-         //          return result.ToArray();
-         //       }
-         // }
-         //BlueEventLog2.instance().assert(false, "Could not find site {0} in config", forSite);
-         return null;
+         return getRouting();
+
       }
 
       public List<string> getAllPubSubs() {

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -304,6 +304,7 @@ namespace BCG {
       }
 
       private void myCollectMissingHeartbeatConnections() {
+         return;  // disabling HBs for now to simplify things for the demo
          if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) {
             myLastHeartbeatDeadCollectionTime = DateTime.Now;
             ///these are all the hbs received bw the last
@@ -321,10 +322,9 @@ namespace BCG {
                if (!indexedHbs.ContainsKey(clientURL))
                   deadClients.Add(clientURL);
             foreach (string deadClient in deadClients) {
-               string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
+               string dispose_msg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
                                                    "connection count before deleting this = {1}", deadClient, myConnections.Count);
-               //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection);
-               Console.WriteLine(disposeMsg);
+               bts.Log.Gen.error(dispose_msg);
                myConnections[deadClient].Dispose();
                myConnections.Remove(deadClient);
                myDeadConnectionURLs.enqueue(deadClient);

src/DataStreamSuite/PubSub/StaticPublisherCore.cs

@@ -181,7 +181,7 @@ namespace BCG {
 
       public void publish(string tagType, string tag, Stream content) {
          PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
-         bts.Log.Gen.info($"StaticPublisherCore.publish msg: {msg}");
+         bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}");
          myMainQueue.enqueue(msg);
       }
 
@@ -357,7 +357,7 @@ namespace BCG {
                myState.publish(tag, tagType, content);
                if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
                   bts.Log.Gen.info("Sending crushable tag: {tag}");
-                  mySendData(tag, tagType, content);
+                  mySendData(tag, tagType, content, "myPublish");
                } else {
                   bts.Log.Gen.info("Could not send tag: {tag}  since its waiting on an ack");
                   myCrushableTagsWaitingOnAck[tag].second++;
@@ -374,8 +374,8 @@ namespace BCG {
       }
 
       // add header, possibly compress the data, and sendto appropriate pubsub
-      private void mySendData(string tag, string tagType, Stream content) {
-         bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType}");
+      private void mySendData(string tag, string tagType, Stream content, string reason) {
+         bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}");
          long initPos = content.Position;
          // locate right pubsub for this tag
          IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
@@ -407,7 +407,6 @@ namespace BCG {
             serverConn.send_with_confirmation(ps_msg, ref refId);
             myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
          } else {
-            //Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
             // this is where the data is actually sent out
             var ps_msg = new PubSubMessage(pub_md, contentToPublish);
             bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
@@ -440,6 +439,7 @@ namespace BCG {
       ///in case of the server coming up, it publishes all the stored state 
       ///for that server, back to it
       protected void myHandleServerReboot(string serverURL, bool connection) {
+         bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}");
          if (connection) {
             string thisPublisherInfo = ClientInfo.thisClient.processString;
             foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
@@ -448,7 +448,7 @@ namespace BCG {
                   string tag = p.Key;
                   myCrushableTagsWaitingOnAck.Clear();
                   byte[] data = myState.getContent(tag, out tagType);
-                  mySendData(tag, tagType, new MemoryStream(data));
+                  mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot");
 
                   //Stream content = new MemoryStream(data);
                   //Stream contentToPublish = content;

src/DataStreamSuite/PubSub/StaticSubscriberCore.cs

@@ -13,8 +13,7 @@ using mbts;
 using mbts.spark;
 //using BCG.Utilities.BlueConfiguration;
 
-namespace BCG
-{
+namespace BCG {
 
    /// <summary>
    /// This class serves as the subscrition interface to the pubsub. The only methods
@@ -25,70 +24,56 @@ namespace BCG
    /// to the servers even though there might be so created by other instances of the SubscriberCore
    /// 
    /// </summary>
-   public class StaticSubscriberCore : ISubscriberCore
-   {
+   public class StaticSubscriberCore : ISubscriberCore {
       private Dictionary<string, ISingleServerSubscriberCore> myServerURLToSubCore = new Dictionary<string, ISingleServerSubscriberCore>();
       private IPubSubRouter myRouter;
       private HighPerformanceTimer myTimer = new HighPerformanceTimer();
       private PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(true);
 
       public event PubSubConnectionChange onPubSubConnectionChange;
-      
-      
+
+
       #region constructors
 
-      public StaticSubscriberCore()
-      {
+      public StaticSubscriberCore() {
          //this defaults to a basic pubsubrouter if the config key is not filled out
          myInit(myGetRouter());
       }
-      public StaticSubscriberCore(string siteName)
-      {
+      public StaticSubscriberCore(string siteName) {
          myInit(new PubSubRouter(PubSub3Config.instance.getRouting(siteName)));
       }
 
-      private IPubSubRouter myGetRouter()
-      {return new PubSubRouter(PubSub3Config.instance.getRouting());}
+      private IPubSubRouter myGetRouter() { return new PubSubRouter(PubSub3Config.instance.getRouting()); }
 
-      public StaticSubscriberCore(PubSubRoute[] routes)
-      {
+      public StaticSubscriberCore(PubSubRoute[] routes) {
          myInit(new PubSubRouter(routes));
       }
 
-      public StaticSubscriberCore(IPubSubRouter router)
-      {
+      public StaticSubscriberCore(IPubSubRouter router) {
          myInit(router);
       }
 
       #endregion
 
-      private void myInit(IPubSubRouter router)
-      {
+      private void myInit(IPubSubRouter router) {
          myRouter = router;
       }
 
-      public ArrayList getUpdates()
-      {
+      public ArrayList getUpdates() {
          return getUpdates(10);
       }
 
-      public ArrayList getUpdates(TimeSpan timeout)
-      {
+      public ArrayList getUpdates(TimeSpan timeout) {
          return getUpdates(Convert.ToInt32(timeout.TotalMilliseconds));
       }
 
-      public ArrayList getUpdates(int timeout)
-      {
+      public ArrayList getUpdates(int timeout) {
          //BCG.BlueTrace.WriteTraceMethodEnterW("StaticSubscriberCore.getUpdates");
-         lock (myServerURLToSubCore)
-         {
+         lock (myServerURLToSubCore) {
 
-            if (myServerURLToSubCore.Count == 0)
-            {
+            if (myServerURLToSubCore.Count == 0) {
                return new ArrayList();
-            }
-            else
-            {
+            } else {
                ///gets a time share for every subscriber and
                ///calls their updates
                myTimer.start();
@@ -96,6 +81,7 @@ namespace BCG
                List<PubSubUpdate> retVal = new List<PubSubUpdate>();
                foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
                   retVal.AddRange(subCore.getUpdates(timeShare));
+               _log_n_updates(retVal);
                myTimer.stop();
                //if (myTimer.lastIntervalTime > 10)
                //   BlueEventLog.logEvent("get updates taking " + myTimer.lastIntervalTime, BlueEventLevel.Error);
@@ -104,30 +90,36 @@ namespace BCG
                return new ArrayList(retVal);
             }
          }
-         
+
       }
 
-      private Dictionary<string,bool> myBlueCommTags = new Dictionary<string,bool> ();
-      
-      bool myIsBlueCommTag(string tag)
-      {
-            return false;
+      private IDictionary<string, int> _n_update_map = new Dictionary<string, int>();
+      private void _log_n_updates(IEnumerable<PubSubUpdate> updates) {
+         foreach (PubSubUpdate u in updates) {
+            if (!_n_update_map.ContainsKey(u.tag)) {
+               _n_update_map.Add(u.tag, 0);
+            }
+            _n_update_map[u.tag] = _n_update_map[u.tag] + 1;
+         }
+         // log it now
+         foreach (var kv in _n_update_map) {
+            bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}");
+         }
       }
 
-      public void subscribe(params string[] tags)
-      {
-         lock (myServerURLToSubCore)
-         {
-            foreach (string tag in tags)
-            {
-               if (myIsBlueCommTag(tag))
-               {
-               }
-               else
-               {
+      private Dictionary<string, bool> myBlueCommTags = new Dictionary<string, bool>();
+
+      bool myIsBlueCommTag(string tag) {
+         return false;
+      }
+
+      public void subscribe(params string[] tags) {
+         lock (myServerURLToSubCore) {
+            foreach (string tag in tags) {
+               if (myIsBlueCommTag(tag)) {
+               } else {
                   string serverURL = myRouter.getSubURLForTag(tag);
-                  if (!myServerURLToSubCore.ContainsKey(serverURL))
-                  {
+                  if (!myServerURLToSubCore.ContainsKey(serverURL)) {
                      // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
                      myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
                      myServerURLToSubCore[serverURL].onPubSubConnectionChange += myHandleConnectionChange;
@@ -138,28 +130,22 @@ namespace BCG
             }
          }
       }
-      public void myHandleConnectionChange (bool isConnected, string connection)
-      {
+      public void myHandleConnectionChange(bool isConnected, string connection) {
          if (onPubSubConnectionChange != null)
             onPubSubConnectionChange(isConnected, connection);
 
       }
 
-      public void unsubscribe(params string[] tags)
-      {
-         lock (myServerURLToSubCore)
-         {
+      public void unsubscribe(params string[] tags) {
+         lock (myServerURLToSubCore) {
 
-            foreach (string tag in tags)
-            {
+            foreach (string tag in tags) {
                if (myIsBlueCommTag(tag))
                   myServerURLToSubCore["bluecomm"].unsubscribe(tag);
-               else
-               {
+               else {
                   string serverURL = myRouter.getSubURLForTag(tag);
 
-                  if (!myServerURLToSubCore.ContainsKey(serverURL))
-                  {
+                  if (!myServerURLToSubCore.ContainsKey(serverURL)) {
                      // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
                      myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
                   }
@@ -168,15 +154,13 @@ namespace BCG
             }
          }
       }
-      public IPubSubRouter getRouter()
-      {
+      public IPubSubRouter getRouter() {
          return myRouter;
       }
 
       #region IDisposable Members
 
-      public void Dispose()
-      {
+      public void Dispose() {
 
          foreach (ISingleServerSubscriberCore sssc in myServerURLToSubCore.Values)
             sssc.Dispose();

src/DataStreamSuite/PubSub/StreamOpCode.cs

@@ -3,10 +3,8 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG
-{
-   public class StreamOpCode
-   {
+namespace BCG {
+   public class StreamOpCode {
       public static readonly byte create = 0,            //creates a table
                                     add = 1,             //adds a single row
                                     remove = 2,          //removes a single row

src/DataStreamSuite/PubSub/SubscriberCore.cs

@@ -4,82 +4,68 @@ using System.Linq;
 using System.Text;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
    /*
     *  This class has been changed to be a thin pass-through shell around different subscriber core implementations.
     *  The specific implementation used is determined by the config. 
     * 
     * 
     * */
-   public class SubscriberCore  : ISubscriberCore
-   {
+   public class SubscriberCore : ISubscriberCore {
       ISubscriberCore mySubscriberCore = null;
 
-      public SubscriberCore()
-      {
+      public SubscriberCore() {
          // TODOds
          mySubscriberCore = new StaticSubscriberCore();//  PubSubComponentFactory.getSubscriberCore();
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
 
-      public SubscriberCore(string siteName)
-      {
+      public SubscriberCore(string siteName) {
          mySubscriberCore = new StaticSubscriberCore(siteName);
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
-      public SubscriberCore(IPubSubRouter theRouter)
-      {
+      public SubscriberCore(IPubSubRouter theRouter) {
          // TODOds
-         mySubscriberCore = new SubscriberCore(theRouter);//  PubSubComponentFactory.getSubscriberCore();
+         mySubscriberCore = new StaticSubscriberCore(theRouter);//  PubSubComponentFactory.getSubscriberCore();
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
 
-      public SubscriberCore(PubSubRoute[] routes)
-      {
+      public SubscriberCore(PubSubRoute[] routes) {
          //this is always a static subscriber core
          mySubscriberCore = new StaticSubscriberCore(routes);
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
 
 
-      public IPubSubRouter getRouter()
-      {
+      public IPubSubRouter getRouter() {
          return mySubscriberCore.getRouter();
       }
 
-      public ArrayList getUpdates()
-      {
+      public ArrayList getUpdates() {
          return mySubscriberCore.getUpdates(10);
       }
 
-      public ArrayList getUpdates(TimeSpan timeout)
-      {
+      public ArrayList getUpdates(TimeSpan timeout) {
          return mySubscriberCore.getUpdates(Convert.ToInt32(timeout.TotalMilliseconds));
       }
 
-      public ArrayList getUpdates(int timeout)
-      {
+      public ArrayList getUpdates(int timeout) {
          return mySubscriberCore.getUpdates(timeout);
       }
 
-      public void subscribe(params string[] tags)
-      {
+      public void subscribe(params string[] tags) {
          mySubscriberCore.subscribe(tags);
       }
 
-      public void unsubscribe(params string[] tags)
-      {
+      public void unsubscribe(params string[] tags) {
          mySubscriberCore.unsubscribe(tags);
       }
 
-      public void Dispose()
-      {
+      public void Dispose() {
          mySubscriberCore.Dispose();
 
       }
-      private void myHandleConnectionChange(bool isConnected, string connection)
-      {
+      private void myHandleConnectionChange(bool isConnected, string connection) {
          if (onPubSubConnectionChange != null)
             onPubSubConnectionChange(isConnected, connection);
       }

src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs

@@ -102,14 +102,6 @@ namespace BCG
             System.Threading.Thread.Sleep(1000);
          }
 
-         // Task.Run(async () =>
-         // {
-         //    while (!_conn.isDisposed)
-         //    {
-         //       await _conn.send_async(buff);
-         //       await Task.Delay(100);
-         //    }
-         // });
       }
 
       public void onData(RemoteMessage msg)

src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs

@@ -0,0 +1,124 @@
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("bbo")]
+   public class BBOStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
+      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("bid_price", TypeCode.Double),
+                                                Tuple.Create("bid_sz", TypeCode.Single),
+                                                Tuple.Create("ask_price", TypeCode.Double),
+                                                Tuple.Create("ask_sz", TypeCode.Single));
+
+      // TODO: add cons for symbol, exchange etc 
+      public BBOStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public BBOStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("BBOStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.books().Subscribe(_on_books);
+
+      }
+
+      private void _log_updates(TableUpdate<BookTuple, uint> update) {
+         int i = 0;
+         foreach (BookTuple bt in update.added) {
+            bts.Log.Gen.info($"BBOStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         }
+         foreach (Change<BookTuple> c in update.changes) {
+            var bt = c.New;
+            bts.Log.Gen.info($"BBOStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         }
+      }
+
+      private void _on_books(TableUpdate<BookTuple, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"BBOStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private void _scan_bbo() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("BBOStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_bbo_schema);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            BookTuple bt = kv.Value;
+            IBlueRow r = new BlueRow(_bbo_schema, id, skey, bt.book.bids[0].price, bt.book.bids[0].size,
+                                     bt.book.asks[0].price, bt.book.asks[0].size);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _scan_bbo();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs

@@ -0,0 +1,146 @@
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("books")]
+   public class BookStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
+      MarketDataClient _md_client;
+      HighPerformanceTimer _timer = new HighPerformanceTimer();
+      private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("level", TypeCode.UInt32),
+                                                Tuple.Create("bid_price", TypeCode.Double),
+                                                Tuple.Create("bid_sz", TypeCode.Single),
+                                                Tuple.Create("ask_price", TypeCode.Double),
+                                                Tuple.Create("ask_sz", TypeCode.Single));
+
+      // TODO: add cons for symbol, exchange etc 
+      public BookStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public BookStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("BookStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.books().Subscribe(_on_books);
+
+      }
+
+      private void _log_updates(TableUpdate<BookTuple, uint> update) {
+         int i = 0;
+         foreach (BookTuple bt in update.added) {
+            bts.Log.Gen.info($"BookStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         }
+         foreach (Change<BookTuple> c in update.changes) {
+            var bt = c.New;
+            bts.Log.Gen.info($"BookStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         }
+      }
+
+      private void _on_books(TableUpdate<BookTuple, uint> update) {
+         _log_updates(update);
+         bts.Log.Gen.info($"BookStream._on_books state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private Dictionary<SecurityKey, Dictionary<int, int>> _id_map = new Dictionary<SecurityKey, Dictionary<int, int>>();
+      private int _get_id(SecurityKey skey, int depth) {
+         Dictionary<int, int> d_map = null;
+         if (!_id_map.TryGetValue(skey, out d_map)) {
+            d_map = new Dictionary<int, int>();
+            d_map.Add(depth, _id++);
+            _id_map.Add(skey, d_map);
+         }
+         int rv = 0;
+         if (!d_map.TryGetValue(depth, out rv)) {
+            rv = _id++;
+            d_map.Add(depth, rv);
+         }
+         return rv;
+      }
+
+      private void _scan_bbo() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("BookStream._scan_bbo");
+         _timer.start();
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_bbo_schema);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            BookTuple bt = kv.Value;
+            for (int level = 0; level < bt.book.bids.Count; level++) {
+               int id = _get_id(skey, level);
+               IBlueRow r = new BlueRow(_bbo_schema, id, skey, Convert.ToUInt32(level), bt.book.bids[level].price, bt.book.bids[0].size,
+                                     bt.book.asks[level].price, bt.book.asks[level].size);
+               new_state.addRow(r);
+            }
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         _timer.stop();
+         bts.Log.Gen.info($"BookStream._scan_bbo time: {_timer.lastIntervalTime} msecs");
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _scan_bbo();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs

@@ -0,0 +1,120 @@
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.md;
+
+   using Spark.Core.Reactive;
+
+   using System;
+
+   using System.Collections.Generic;
+
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   using SecurityKey = System.UInt32;
+
+   [DataStreamUserRespresentation("instruments")]
+   public class InstrumentsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<uint, SecurityImmutable> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private readonly IBlueTableSchema _instrument_schema = BlueDataUtility.form_schema("instruments",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("sec_type", TypeCode.String),
+                                                Tuple.Create("venue", TypeCode.String),
+                                                Tuple.Create("symbol", TypeCode.String),
+                                                Tuple.Create("expiration", TypeCode.DateTime),
+                                                Tuple.Create("strike", TypeCode.Double),
+                                                Tuple.Create("callput", TypeCode.String)
+                                                );
+
+      // TODO: add cons for symbol, exchange etc 
+      public InstrumentsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public InstrumentsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("InstrumentsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.securities().Subscribe(_on_instruments);
+
+      }
+
+      private void _log_updates(IReadOnlyDictionary<uint, SecurityImmutable> state) {
+         foreach (SecurityImmutable security in state.Values) {
+            bts.Log.Gen.info($"InstrumentsStream sec: {security.ToString()}");
+         }
+      }
+
+      private void _on_instruments(IReadOnlyDictionary<uint, SecurityImmutable> state) {
+         _dirty = true;
+         _log_updates(state);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count()}");
+         _cached_state = state;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private void _scan_instruments() {
+         // this shouldn be made incremental using the TableUpdate
+         var count = _cached_state == null ? 0 : _cached_state.Count;
+         bts.Log.Gen.info($"InstrumentsStream._scan_instruments _cached_state: {count}");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_instrument_schema);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            SecurityImmutable sec = kv.Value;
+            bts.api.SecurityType sec_type = sec.security_type();
+            object exp = DBNull.Value;
+            if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
+               exp = sec.security_expiration();
+            object strike = DBNull.Value;
+            if (sec_type == bts.api.SecurityType.Option)
+               strike = sec.strike();
+            object cp = DBNull.Value;
+            if (sec_type == bts.api.SecurityType.Option)
+               cp = sec.Option.CallPut.ToString();
+            IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, exp, strike, cp);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_instrument_schema));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan_instruments();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -0,0 +1,125 @@
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("price_stats")]
+   public class PriceStatsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.Int32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("open", TypeCode.UInt32),
+                                                Tuple.Create("close", TypeCode.Double),
+                                                Tuple.Create("settlement", TypeCode.Single));
+
+      // TODO: add cons for symbol, exchange etc 
+      public PriceStatsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public PriceStatsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("PriceStatsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         _md_client.price_stats().Subscribe(_on_price_stats);
+      }
+
+      private void _log_updates(TableUpdate<PriceStats, uint> update) {
+         int i = 0;
+         // foreach (BookTuple bt in update.added) {
+         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+         // foreach (Change<BookTuple> c in update.changes) {
+         //    var bt = c.New;
+         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+      }
+
+      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+         _dirty = false;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+      }
+
+      private void _scan() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_pstats_scheam);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            PriceStats pstats = kv.Value;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs

@@ -0,0 +1,162 @@
+namespace BCG
+{
+   using bts.api.orderman;
+
+   enum TSGroupByOperation
+   {
+      OHLC = 0,
+      Statistics
+   }
+
+   [DataStreamUserRespresentation("timeseriesanalysis")]
+   public class TimeSeriesGroupByStream : ViewStream
+   {
+      public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation)
+      {
+         ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
+         mySetRealSource(rs);
+      }
+   }
+   public class OHLC
+   {
+      public OHLC(double open_args, double high_args, double low_args, double close_args)
+      {
+         open = open_args;
+         high = high_args;
+         low = low_args;
+         close = close_args;
+      }
+      public readonly double open, high, low, close;
+   }
+   public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator
+   {
+      private string _timestamp_col;
+      private string _output_col;
+      private IBlueTableSchema _out_schema;
+      public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col)
+      {
+         _timestamp_col = timestamp_col;
+         _output_col = output_col;
+      }
+
+      string open_col
+      {
+         get
+         {
+            return $"{_output_col}_open"; ;
+         }
+      }
+      string high_col
+      {
+         get
+         {
+            return $"{_output_col}_high"; ;
+         }
+      }
+      string low_col
+      {
+         get
+         {
+            return $"{_output_col}_low"; ;
+         }
+      }
+      string close_col
+      {
+         get
+         {
+            return $"{_output_col}_close"; ;
+         }
+      }
+      private void _build_schema(IBlueTableSchema input_schema)
+      {
+         List<Tuple<Type, string>> new_cols = new List<Tuple<Type, string>>();
+         ColumnInfo ci = input_schema[_output_col];
+         new_cols.Add(Tuple.Create(ci.colType, open_col));
+         new_cols.Add(Tuple.Create(ci.colType, high_col));
+         new_cols.Add(Tuple.Create(ci.colType, low_col));
+         new_cols.Add(Tuple.Create(ci.colType, close_col));
+         // should hide the output col
+         _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
+      {
+         if (_out_schema == null)
+         {
+            if (grouped_rows.Count > 0)
+            {
+               foreach (var kv in grouped_rows)
+               {
+                  var rows = kv.Value;
+                  if (rows != null && rows.Count > 0)
+                  {
+                     _build_schema(rows.ElementAt(0).schema);
+                  }
+               }
+            }
+         }
+         BlueTable rv = new BlueTable(_out_schema);
+         foreach (var kv in grouped_rows)
+         {
+            int id = kv.Key;
+            IEnumerable<IBlueRow> rows = kv.Value;
+            // sort rows on time to compute o,c correctly??
+            if (rows != null && rows.Count() > 0)
+            {
+               IBlueRow reprow = rows.ElementAt(0);
+               IEnumerable<double> vals = BlueDataUtility.get_covalues<double>(rows, _output_col);
+               OHLC ohlc = compute_ohlc(vals);
+               MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow);
+               nr.setValue("id", kv.Key);
+               nr.setValue(open_col, ohlc.open);
+               nr.setValue(high_col, ohlc.high);
+               nr.setValue(low_col, ohlc.low);
+               nr.setValue(close_col, ohlc.close);
+               rv.addRow(nr);
+            }
+         }
+         return rv;
+      }
+
+      public string[] get_group_by_cols()
+      {
+         return new string[] { _timestamp_col };
+      }
+
+      static OHLC compute_ohlc(IEnumerable<double> prices)
+      {
+         double o = prices.ElementAt(0);
+         double c = prices.ElementAt(prices.Count() - 1);
+         double l = prices.Min();
+         double h = prices.Max();
+         return new(o, h, l, c);
+      }
+   }
+   [DataStreamUserRespresentation("RSI")]
+   public class RSIStream : ViewStream
+   {
+      // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
+      public RSIStream(ViewStream md_str, string timebucket_column, string output_col)
+      {
+
+      }
+   }
+   [DataStreamUserRespresentation("Bollinger")]
+   public class BollingerBandStream : ViewStream
+   {
+      // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
+      public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col)
+      {
+      }
+   }
+   [DataStreamUserRespresentation("OHLC")]
+   public class OHLCStream : ViewStream
+   {
+      // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
+      public OHLCStream(ViewStream md_str, string timebucket_column, string output_col)
+      {
+         ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
+         mySetRealSource(rs);
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs

@@ -0,0 +1,113 @@
+namespace BCG
+{
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.md;
+
+   using mbts.spark;
+
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+
+   using System.Linq;
+   using System.Reactive.Linq;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+
+   [DataStreamUserRespresentation("timesales")]
+   public class TimeSalesStream : ViewStream, IWorker
+   {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<uint, SecurityImmutable> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private ConcurrentQueue<TradeInfo> _cached_trades = new ConcurrentQueue<TradeInfo>();
+      private readonly int _buffer_size;
+      private IBlueTableSchema _schema = BlueDataUtility.form_schema("TimeSales",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("time", TypeCode.DateTime),
+                                                Tuple.Create("price", TypeCode.Double),
+                                                Tuple.Create("qty", TypeCode.UInt32),
+                                                Tuple.Create("aggressor", TypeCode.String));
+      public TimeSalesStream(int buffer_size)
+      : this(buffer_size, ViewStreamDatabase.instance)
+      {
+      }
+
+      public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd)
+      {
+         _buffer_size = buffer_size;
+         bts.Log.Gen.info("TimeSalesStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _vsd.addWorker(this);
+         _md_client.trades(new TradesSubscriptionFilter()).Subscribe(_on_trades);
+         //_md_client.trades(null).Subscribe(_on_trades);
+
+      }
+
+      private void _log_updates(IList<TradeInfo> trades)
+      {
+         bts.Log.Gen.info($"TimeSalesStream update sz: {trades.Count}");
+      }
+
+      private void _on_trades(IList<TradeInfo> trades)
+      {
+         _dirty = true;
+         _log_updates(trades);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"TimeSalesStream._on_trades trades: {trades.Count()}");
+         foreach (var t in trades)
+            _cached_trades.Enqueue(t);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         _vsd.removeWorker(this);
+      }
+
+      private void _scan_trades()
+      {
+         Func<TradeInfo, DateTime> to_dt = (tinf) =>
+         {
+            return new mbts.utils.LLDateTime((ulong)tinf.trade.Time).to_datetime().ToLocalTime();
+         };
+
+         List<TradeInfo> trades = new List<TradeInfo>(_cached_trades);
+         var latest_n_trades = trades.OrderByDescending(t => (ulong)t.trade.Time).Take(_buffer_size).ToList();
+         BlueTable new_state = new BlueTable(_schema);
+         foreach (TradeInfo tinf in Enumerable.Reverse(latest_n_trades))
+         {
+            // TODO, time
+            IBlueRow r = new BlueRow(_schema, _id++, tinf.security.Id, tinf.security.ToString(), to_dt(tinf), tinf.trade.Mantissa * Math.Pow(10, tinf.trade.Exponent), tinf.trade.Size, tinf.trade.AggressorSide.ToString());
+            new_state.addRow(r);
+         }
+         DataPacket dp = BlueDataUtility.build_dp_inc(new_state, myLastValue as DataPacket);
+         myFireEvent(dp);
+      }
+
+      public void doWork()
+      {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period)
+         {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan_trades();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\PubSub\PubSub.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+  <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/SharedStreams/System/PS3/Health/DeadPubSubServerTagsStream.cs

@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// List of all the tags that become 'dead'
+//    /// if a pubsub dies. this uses the pubsub heartbeat
+//    /// to detect the health of the pubsub server.
+//    /// </summary>
+//    [DataStreamUserRespresentation("deadPubSubTags")]
+//    public class DeadPubSubServerTagsStream : ViewStream, IStreamMapper
+//    {
+//       public DeadPubSubServerTagsStream()
+//          : this (false)
+//       {
+//       }
+
+//       public DeadPubSubServerTagsStream(bool publishedTagsOnly)
+//       {
+//          ViewStream tags = new PubSubServerStateStream();
+//          if (publishedTagsOnly)
+//             tags = new FilterStream(tags, "publishers<>'' or deadPublishers<>''");
+//          ViewStream deadPubSubs = new FilterStream(new PubSubServerHealthStream(),
+//                                                       "status='notOK'");
+//          deadPubSubs = new MapStream(deadPubSubs, this);
+//          mySetRealSource(new JoinStream(tags, "server", "!id subscribers publishers deadPublishers",
+//                                           deadPubSubs, "serverURL", "!id lastHeartbeat status serverURL"));
+//       }
+
+
+//       #region IStreamMapper Members
+
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {return inputSchema;}
+
+//       #endregion
+
+//       #region IDynamicStreamMapper Members
+
+//       public IBlueRow map(IBlueRow row)
+//       {
+//          string ip = row["serverURL"].ToString().Split(':')[1].Substring(2);
+//          return BlueDataUtility.getNewReplacedRow(row, ip, "serverURL");
+//       }
+
+//       #endregion
+//    }
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Health/DeadPublisherTagsStream.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BCG
+{
+   /// <summary>
+   /// List of all the tags that result from a
+   /// dead publisher, ie tags that have data
+   /// but no active publisher
+   /// </summary>
+   [DataStreamUserRespresentation ("deadPublisherTags")]
+   public class DeadPublisherTagsStream : ViewStream
+   {
+      public DeadPublisherTagsStream()
+      {
+         ViewStream serverState = new PubSubServerStateStream();
+         serverState = new FilterStream(serverState, "publishers='' and deadPublishers<>''");
+         serverState = new ProjectStream(serverState, "!subscribers publishers deadPublishers");
+         mySetRealSource(serverState);
+      }
+
+   }
+}

src/DataStreamSuite/SharedStreams/System/PS3/Health/DeadTagsStream.cs

@@ -0,0 +1,23 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+
+// namespace BCG
+// {
+//    /// <summary>
+//    /// list of all the 'dead' tags, where a tag is defined
+//    /// as dead when either the pubsub dies or the publisher
+//    /// dies. however to detect to dead pubsubs, this stream
+//    /// has to be constructed when the pubsub was alive (or
+//    /// else it would have no source of meta data information)
+//    /// </summary>
+//    [DataStreamUserRespresentation ("deadTags")]
+//    public class DeadTagsStream : ViewStream
+//    {
+//       public DeadTagsStream()
+//       {
+//          mySetRealSource(new AppendStream(new DeadPubSubServerTagsStream(true),
+//                                              new DeadPublisherTagsStream()));
+//       }
+//    }
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubLatencyStream.cs

@@ -0,0 +1,232 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Collections;
+// using System.Linq;
+// using System.Text;
+
+// using BCG.Utilities;
+// using BCG.Utilities.General;
+// using System.IO;
+
+// namespace BCG
+// {
+//    /*
+//     *       Connects to a given pubsub and publishes data there, then measures the time it takes for the data to arrive. 
+//     *       Becuase it publishes data to a pubsub, it implements ISideEffectStream.
+//     *       The messages it sends to the pubsub are really simple, just a GUID and a timestamp. The timestamp
+//     *       isn't really even necessary, but it helped with debugging and it's just 8 bytes. 
+//     *      
+//     * */
+//    public class PubSubLatencyStream : ViewStream, ISideEffectStream, IWorker
+//    {
+//       private PublisherCore myPubCore;
+//       private SubscriberCore mySubCore;
+//       private IBlueTableSchema mySchema;
+
+//       private ViewStreamDatabase myVSD;
+
+//       //for debugging
+//       private string myURL;
+
+//       private TimeSpan myTestPeriod;
+//       private DateTime myNextWorkTime;
+
+//       private int myTimeout;
+//       private static string tagType = "SystemInfo_LatencyTest";
+//       private static string tagName = "SystemInfo_LatencyTest";
+
+
+//       //this map says what time updates were sent out at
+//       //if an update is recievd, it is removed from this map
+//       private Dictionary<Guid, DateTime> myUnrespondedUpdates;
+
+//       public PubSubLatencyStream() : this(ViewStreamDatabase.instance, 5000,5000) {}
+//       public PubSubLatencyStream(int period, int timeout): this(ViewStreamDatabase.instance, period, timeout) {}
+
+
+//       public PubSubLatencyStream(ViewStreamDatabase vsd,int period, int timeOut)
+//       {
+//          if(PubSub3Config.instance.PS3_UseSubstations)
+//          {
+//             SubstationServerBaseStream theRealSource = new SubstationServerBaseStream((service, url) =>
+//                {
+//                   return new PubSubLatencyStream(vsd, url);
+//                });
+
+//             theRealSource.addAllSubstations(false);
+
+//             mySetRealSource(new ThreadSafeStream(theRealSource));
+//          }
+//          else
+//          {
+//             //subscribe to each of the pubsub routes in the routing table
+//             List<ViewStream> theRealSources = new List<ViewStream>();
+
+//             foreach (string url in PubSub3Config.instance.getAllPubSubs())
+//                theRealSources.Add( new PubSubLatencyStream(vsd,url,period,timeOut));
+
+//             mySetRealSource(new AppendStream(theRealSources.ToArray()));
+
+//          }
+
+//       }
+
+//       public PubSubLatencyStream(string urlOrService) : this(ViewStreamDatabase.instance, urlOrService, 5000, 5000) { }
+//       public PubSubLatencyStream(ViewStreamDatabase vsd, string urlOrService) : this(vsd,urlOrService,5000,5000) {}
+
+//       public PubSubLatencyStream(string urlOrService, int testPeriodMS, int timeOut) : this(ViewStreamDatabase.instance, urlOrService, testPeriodMS, timeOut) { } 
+//       //arguments: a VSD to work with, the url of hte pubsub being tested, how frequently to send updates, and how long to wait before assuming an update was lost
+//       public PubSubLatencyStream(ViewStreamDatabase vsd, string urlOrService, int testPeriodMS, int timeOut)
+//       {
+
+//          int colNo = 0;
+
+//          mySchema = new BlueTableSchema("pubsub latency: " + urlOrService, new ColumnInfo("id", colNo++, typeof(int)),
+//                                                                   new ColumnInfo("url", colNo++, typeof(string)),
+//                                                                   new ColumnInfo("latencyMs", colNo++, typeof(double)));
+
+//          myUnrespondedUpdates = new Dictionary<Guid, DateTime>();
+
+//          myTestPeriod = new TimeSpan(0, 0, 0, 0, testPeriodMS);
+//          myNextWorkTime = DateTime.Now;
+//          myTimeout = timeOut;
+
+//          myVSD = vsd;
+//          myURL = urlOrService;
+
+
+//          if (urlOrService.Contains("bluto://"))
+//          {
+//             myPubCore = new PublisherCore(urlOrService);
+//             mySubCore = new SubscriberCore(new PubSubRouter(new PubSubRoute(myURL, ".*")));
+//             mySubCore.subscribe(tagName);
+//          }
+//          else
+//          {
+//             //we were give a service name. We need to connect to that specific service. 
+//             SubstationServerBaseStream theRealSource = new SubstationServerBaseStream((service, url) => { return new PubSubLatencyStream(vsd, url, testPeriodMS, timeOut); });
+//             theRealSource.addSubstation(urlOrService);
+//             mySetRealSource(new ThreadSafeStream(theRealSource));
+//          }
+
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myVSD.addWorker(this);
+
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          myVSD.removeWorker(this);
+//       }
+
+//       private void mySendUpdateMessage()
+//       {
+//          MemoryStream memStream = new MemoryStream();
+//          BinaryWriter bw = new BinaryWriter(memStream);
+
+//          Guid theGuid = Guid.NewGuid();
+//          DateTime now = DateTime.Now;
+
+//          //publish the update
+//          bw.Write(theGuid.ToByteArray());
+//          bw.Write(now.ToOADate());
+//          memStream.Position = 0;
+//          myPubCore.publish(tagType, tagName, memStream);
+
+//          //record the fact that we published the update
+//          myUnrespondedUpdates.Add(theGuid, now);
+//       }
+
+
+
+//       private void myHandleUpdate(Stream update)
+//       {
+
+//          int guidSize = Guid.NewGuid().ToByteArray().Length;
+//          byte[] thisMsgGuidBytes = new byte[guidSize];
+
+//          //read the message
+//          BinaryReader br = new BinaryReader(update);
+//          br.Read(thisMsgGuidBytes, 0, guidSize);
+//          Guid thisMsgGuid = new Guid(thisMsgGuidBytes);
+//          DateTime thisMsgTimestamp = DateTime.FromOADate(br.ReadDouble());
+
+//          //update the state of the table
+//          //sometimes we'll get updates that we haven't subscribed to
+//          //this can happen if this stream is created, then destroyed and recreated really fast
+
+//          if (myUnrespondedUpdates.ContainsKey(thisMsgGuid))
+//          {
+//             //fire off this update 
+//             double msgLatency = (DateTime.Now - thisMsgTimestamp).TotalMilliseconds;
+//             myFireUpdate(msgLatency);
+//             myUnrespondedUpdates.Remove(thisMsgGuid);
+
+//          }
+
+//       }
+
+//       private void myCheckForLostUpdates()
+//       {
+//          DateTime now = DateTime.Now;
+//          double worstLatency = 0.0;
+
+//          foreach (KeyValuePair<Guid, DateTime> kvp in myUnrespondedUpdates)
+//          {
+//             double latency = (now - kvp.Value).TotalMilliseconds;
+//             if (latency> myTimeout)
+//             {
+//                if (latency > worstLatency)
+//                   worstLatency = latency;
+//             }
+
+//          }
+
+//          if (worstLatency != 0)
+//             myFireUpdate(worstLatency);
+
+//       }
+
+//       private void myFireUpdate(double latency)
+//       {
+//              BlueTable updateTable = new BlueTable(mySchema);
+
+//              updateTable.addRow(new BlueRow(mySchema, 0, myURL, latency));
+
+//             if(myLastValue != null)
+//                myFireEvent(new DataPacket(updateTable,updateTable, ((DataPacket)myLastValue).newRows));
+//             else
+//             {
+//                //we have to give them _something_ for the added and deleted rows
+//                BlueTable emptyTable = new BlueTable(mySchema);
+//                myFireEvent(new DataPacket(updateTable, emptyTable,emptyTable));
+//             }
+
+//       }
+
+//       public void doWork()
+//       {
+//          DateTime now = DateTime.Now;
+//          if (now > myNextWorkTime)
+//          {
+//             mySendUpdateMessage();
+//             myNextWorkTime = now + myTestPeriod;
+//          }
+
+//          ArrayList updates = mySubCore.getUpdates();
+
+//          foreach (PubSubUpdate update in updates)
+//          {
+//             myHandleUpdate(update.body);
+//          }
+
+//          myCheckForLostUpdates();
+
+//       }
+
+
+//    }
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs

@@ -0,0 +1,137 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+
+
+   [DataStreamUserRespresentation("tagStats")]
+   public class PubSubServerTagStatsStream : ViewStream, IWorker
+   {
+      private static Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>> myStreamMap =
+         new Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>>();
+
+      private SubscriberCore mySubCore;
+      private ViewStreamDatabase myVsd;
+      private Dictionary<string, IBlueRow> myTagToStatsMap = new Dictionary<string, IBlueRow>();
+      private KeysBasedTable2 myTable = new KeysBasedTable2(new BlueTableSchema("", new ColumnInfo("id", 0, typeof(int)),
+                                                                           new ColumnInfo("server", 1, typeof(string)),
+                                                                           new ColumnInfo("tag", 2, typeof(string)),
+                                                                           new ColumnInfo("nUpdates", 3, typeof(int)),
+                                                                           new ColumnInfo("totalBytes", 4, typeof(int)),
+                                                                           new ColumnInfo("minSize", 5, typeof(int)),
+                                                                           new ColumnInfo("maxSize", 6, typeof(int)),
+                                                                           new ColumnInfo("lastUpdateSize", 7, typeof(int)),
+                                                                           new ColumnInfo("averageSize", 8, typeof(double)),
+                                                                           new ColumnInfo("lastUpdateTime", 9, typeof(DateTime))),
+                                                            new string[] { "tag" });
+
+      public static ViewStream getPubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL)
+      {
+         //you can also supply a service name and it should work, as long as substations are enabled. 
+         // if (PubSub3Config.instance.PS3_UseSubstations)
+         // {
+         //    return new ThreadSafeStream(SubstationTagStatsStream.getSubstationTagStatsStream(vsd, serverURL));
+         // }
+         if (!myStreamMap.ContainsKey(vsd))
+            myStreamMap.Add(vsd, new Dictionary<string, ViewStream>());
+         if (!myStreamMap[vsd].ContainsKey(serverURL))
+            myStreamMap[vsd].Add(serverURL, new PubSubServerTagStatsStream(vsd, serverURL));
+         return myStreamMap[vsd][serverURL];
+      }
+
+      #region constructors to be used by the static factory method
+
+      public PubSubServerTagStatsStream(string serverURL)
+         : this(ViewStreamDatabase.instance, serverURL)
+      { }
+
+      private PubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL)
+      {
+         myVsd = vsd;
+         mySubCore = new SubscriberCore(new PubSubRouter(new PubSubRoute[1] { new PubSubRoute(serverURL, ".*") }));
+      }
+
+      #endregion
+
+      #region exposed constructors that use the static factory methods
+
+      public PubSubServerTagStatsStream()
+      {
+         // if (PubSub3Config.instance.PS3_UseSubstations)
+         //    mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(ViewStreamDatabase.instance, false)));
+         // else
+         mySetRealSource(new PubSubServerTagStatsStream(ViewStreamDatabase.instance, PubSub3Config.instance.getSubRouting()));
+
+      }
+
+      public PubSubServerTagStatsStream(ViewStreamDatabase vsd)
+      {
+         // if (PubSub3Config.instance.PS3_UseSubstations)
+         //    mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(vsd, false)));
+         // else
+         mySetRealSource(new PubSubServerTagStatsStream(vsd, PubSub3Config.instance.getSubRouting()));
+
+      }
+
+      public PubSubServerTagStatsStream(ViewStreamDatabase vsd, PubSubRoute[] routes)
+      {
+         List<ViewStream> vss = new List<ViewStream>();
+         foreach (string url in PubSub3Config.instance.getAllPubSubs())
+            vss.Add(getPubSubServerTagStatsStream(vsd, url));
+         mySetRealSource(new AppendStream(vss.ToArray()));
+      }
+
+      #endregion
+
+      protected override void myOnFirstSubscriber()
+      {
+         mySubCore.subscribe("System_AllStatistics");
+         myVsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         mySubCore.unsubscribe("System_AllStatistics");
+         myVsd.removeWorker(this);
+      }
+
+      #region IWorker Members
+
+      public void doWork()
+      {
+         foreach (PubSubUpdate u in mySubCore.getUpdates(0))
+         {
+            List<TagStatsInfo> tagStats = new List<TagStatsInfo>();
+            BinaryReader br = new BinaryReader(u.body);
+            int n = br.ReadInt32();
+            for (int i = 0; i < n; i++)
+            {
+               tagStats.Add(new TagStatsInfo(u.body));
+            }
+            myHandleStats(tagStats);
+         }
+      }
+
+      #endregion
+
+      private void myHandleStats(List<TagStatsInfo> tagStats)
+      {
+         BlueTable tagStatsRows = new BlueTable(myTable.schema);
+         int id = 0;
+         foreach (TagStatsInfo t in tagStats)
+         {
+            tagStatsRows.addRow(new BlueRow(myTable.schema, id++, t.server, t.tag, t.totalUpdates, t.totalBytes,
+                                                               t.minSize, t.maxSize, t.lastUpdateSize,
+                                                               t.averageSize, new DateTime(t.lastUpdateTime)));
+         }
+         IBlueRowCollection dRows, aRows;
+         myTable.updateMultipleRows(tagStatsRows, out dRows, out aRows);
+         myFireEvent(new DataPacket(myTable, aRows, dRows));
+
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTotalTrafficStream.cs

@@ -0,0 +1,163 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using BCG.Utilities.General;
+
+
+// namespace BCG
+// {
+//    internal class RateWindow
+//    {
+//       private DateTime myStartTime = DateTime.Now;
+//       private bool myEverAdded = false;
+//       private long myTotalValue = 0, myLastValue = -1;
+//       private TimeSpan myWindowPeriod;
+//       private List<Pair<DateTime, long>> myVals = new List<Pair<DateTime, long>>();
+
+//       public RateWindow(TimeSpan windowPeriod)
+//       {
+//          myWindowPeriod = windowPeriod;
+//       }
+
+//       public double tick()
+//       {
+//          int removeItemsCount = 0;
+//          foreach (Pair<DateTime, long> v in myVals)
+//          {
+//             if (DateTime.Now.Subtract(v.first) >= myWindowPeriod)
+//             {
+//                removeItemsCount++;
+//                myTotalValue -= v.second;
+//             }
+//          }
+//          myVals.RemoveRange(0, removeItemsCount);
+//          TimeSpan period = myStartTime - DateTime.Now;
+//          return myTotalValue / (period > myWindowPeriod ? myWindowPeriod : period).TotalSeconds;
+//       }
+
+//       public void add(long latestTotal)
+//       {
+//          if (!myEverAdded)
+//          {
+//             myLastValue = latestTotal;
+//          }
+//          else
+//          {
+//             long currentIncrement = latestTotal - myLastValue; 
+//             myTotalValue += currentIncrement;
+//             myLastValue = latestTotal;
+//             myVals.Add (new Pair<DateTime,long> (DateTime.Now, currentIncrement));
+//          } 
+//       }
+//    }
+
+//    public class PubSubServerTagTrafficStream : ViewStream, IStreamMapper
+//    {
+//       private IBlueTableSchema mySchema = new BlueTableSchema("Stats",
+//          new ColumnInfo("id", 0, typeof(int)),
+//          new ColumnInfo("pubSub", 1, typeof(string)),
+//          new ColumnInfo("tag", 2, typeof(string)),
+//          new ColumnInfo("nUpdates", 3, typeof(int)),
+//          new ColumnInfo("nSubscribers", 4, typeof(int)),
+//          new ColumnInfo("totalBytesPublished", 5, typeof(long)),
+//          new ColumnInfo("totalBytesMoved", 6, typeof(long)));
+
+
+//       public PubSubServerTagTrafficStream()
+//       {
+//          List<ViewStream> vss = new List<ViewStream>();
+//          foreach (string url in PubSub3Config.instance.getAllPubSubs())
+//             vss.Add(new PubSubServerTagTrafficStream(url));
+//          mySetRealSource(new MutableAppendStream(vss));
+//       }
+
+//       public PubSubServerTagTrafficStream(string serverUrl)
+//          : this(ViewStreamDatabase.instance, serverUrl)
+//       { }
+
+//       public PubSubServerTagTrafficStream (ViewStreamDatabase vsd, string serverUrl)
+//       {
+//          PubSubRoute[] routes = new PubSubRoute[] {new PubSubRoute (serverUrl, ".*")};
+//          ViewStream ls = new PubSubServerTagStatsStream (vsd, routes);
+//          ViewStream rs = new TagSynopsisStream(vsd, serverUrl);
+//          ViewStream r = new JoinStream(ls,
+//                                        "tag",
+//                                        "!id tag",
+//                                        rs,
+//                                        "tag",
+//                                        "!id");
+//          r = new MapStream(r, this);
+//          mySetRealSource(r);
+
+//       }
+
+//       #region IStreamMapper Members
+
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { return mySchema; }
+
+//       #endregion
+
+//       #region IDynamicStreamMapper Members
+
+//       public IBlueRow map(IBlueRow row)
+//       {
+//          return new BlueRow(mySchema, row["id"], row["pubSub"], row["tag"],
+//                                        row["nUpdates"], row["subscribers"], Convert.ToInt64(row["totalBytes"]),
+//                                        Convert.ToInt64(row["totalBytes"]) * ((int)row["subscribers"] + 1));
+//       }
+
+//       #endregion
+//    }
+
+//    public class PubSubServerTotalTrafficStream : ViewStream, IGroupByEvaluator
+//    {
+//       private string myPubsub;
+//       private IBlueTableSchema mySchema = new BlueTableSchema("addup",
+//          new ColumnInfo("id", 0, typeof(int)),
+//          new ColumnInfo("pubSub", 1, typeof(string)),
+//          new ColumnInfo("totalSubscribers", 2, typeof(int)),
+//          new ColumnInfo("totalUpdates", 3, typeof(int)),
+//          new ColumnInfo("totalBytesPublished", 4, typeof(long)),
+//          new ColumnInfo("totalBytesMoved", 5, typeof(long)));
+
+//       public PubSubServerTotalTrafficStream(string serverUrl)
+//       {
+//          myPubsub = serverUrl;
+//          ViewStream rs = new GroupByStream(new PubSubServerTagTrafficStream(serverUrl), this);
+//          mySetRealSource(rs);
+//       }
+
+//       public PubSubServerTotalTrafficStream()
+//       {
+//          List<ViewStream> vss = new List<ViewStream>();
+//          foreach (string url in PubSub3Config.instance.getAllPubSubs())
+//          vss.Add(new PubSubServerTotalTrafficStream(url));
+//          mySetRealSource(new MutableAppendStream(vss));
+//       }
+
+//       #region IGroupByEvaluator Members
+
+//       public IBlueRow evaluate (DataPacket packet)
+//       {
+//          int totalSubs = 0, totalUpdates = 0;
+//          long totalBytesPublished = 0, totalBytesMoved = 0;
+//          foreach (IBlueRow r in packet.newRows)
+//          {
+//             totalSubs += (int)r["nSubscribers"];
+//             totalUpdates += (int)r["nUpdates"];
+//             totalBytesPublished += (long)r["totalBytesPublished"];
+//             totalBytesMoved += (long)r["totalBytesMoved"];
+//          }
+//          return new BlueRow(mySchema, 0, myPubsub, totalSubs, totalUpdates, totalBytesPublished, totalBytesMoved);
+//       }
+
+//       public string[] groupByColumns { get { return new string[] { "pubSub" }; } }
+
+//       public void clear() { }
+
+//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return mySchema; }
+
+//       #endregion
+//    }
+
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubTagLoadStream.cs

@@ -0,0 +1,61 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+
+// namespace BCG 
+// {
+//    public class PubSubTagLoadStream : ViewStream
+//    {
+
+//       private string myTagToMatch;
+
+//       public PubSubTagLoadStream() : this(ViewStreamDatabase.instance, null) { }
+//       public PubSubTagLoadStream(string tag) : this(ViewStreamDatabase.instance, tag) { }
+//       public PubSubTagLoadStream(ViewStreamDatabase db, string tag)
+//       {
+
+//          SubstationServerBaseStream baseStream;
+
+//          if(tag == null)
+//             baseStream = new SubstationServerBaseStream((service, url) => { return new PubSubTagLoadStream(url, db); });
+//          else
+//             baseStream = new SubstationServerBaseStream((service, url) => { return new PubSubTagLoadStream(url, tag, db); });
+
+//          if (PubSub3Config.instance.PS3_UseSubstations)
+//             baseStream.addAllSubstations(false);
+//          else
+//             baseStream.addAllRoutingTablePubsubs();
+
+//          myTagToMatch = null;
+
+//          mySetRealSource(baseStream);
+//       }
+
+
+//       public PubSubTagLoadStream(string url, ViewStreamDatabase vsd)
+//       {
+//          ViewStream tagTrafficStream = new PubSubServerTagTrafficStream(vsd, url);
+//          ViewStream timeSliceStream = new TimeSliceStream(url, vsd);
+
+//          mySetRealSource(new JoinStream(tagTrafficStream, "tag", "!tag id", timeSliceStream, "tag", "!id"));
+//          myTagToMatch = null;
+//       }
+
+//       public PubSubTagLoadStream(string url, string tag, ViewStreamDatabase vsd)
+//       {
+//          ViewStream tagTrafficStream = new FilterStream(new PubSubServerTagTrafficStream(vsd, url), myMatchTag);
+//          ViewStream timeSliceStream = new FilterStream(new TimeSliceStream(url, vsd), myMatchTag);
+
+//          myTagToMatch = tag;
+//          mySetRealSource(new JoinStream(tagTrafficStream, "tag", "!tag id", timeSliceStream, "tag", "!id"));
+//       }
+
+//       private bool myMatchTag(IBlueRow r)
+//       {
+//          return myTagToMatch == r["tag"].ToString();
+//       }
+
+
+//    }
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/AllTagsStream.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections;
+using System.IO;
+
+using BCG.Utilities.General;
+using BCG.Utilities;
+
+namespace BCG
+{
+
+   public class AllTagsStream : ViewStream, IWorker
+   {
+      private IWorker myRealWorkerStream;
+      public AllTagsStream()
+         : this ("")
+      { }
+
+      public AllTagsStream(string prefix)
+         : this (prefix, ViewStreamDatabase.instance)
+      { }
+
+      public AllTagsStream(ViewStreamDatabase vsd, string regExp)
+      {
+         ViewStream rs;
+         mySetRealSource(rs = new TagSynopsisStream (vsd, regExp, true));
+         myRealWorkerStream = rs as IWorker;
+      }
+
+      public AllTagsStream(string prefix, ViewStreamDatabase db)
+         : this (prefix, db, db.subscriber.getRouter())
+      { }
+
+      public AllTagsStream(string prefix, ViewStreamDatabase db, IPubSubRouter router)
+      {
+         ViewStream rs = null;
+         //if we're using substations, we don't care about the router
+         if (PubSub3Config.instance.PS3_UseSubstations)
+            mySetRealSource(rs = new TagSynopsisStream(db,"^" + prefix + ".*",true));
+         else
+            mySetRealSource(rs = new TagSynopsisStream(db, router.getAllPubSubSubURLs(), "^" + prefix + ".*", true));
+         myRealWorkerStream = rs as IWorker;
+      }
+
+      #region IWorker Members
+
+      public void doWork()
+      {
+         if(myRealWorkerStream != null)
+            myRealWorkerStream.doWork();
+      }
+
+      #endregion
+   }
+
+}

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/MissingPubSubDataViewStream.cs

@@ -0,0 +1,38 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+
+// namespace BCG
+// {
+//    [DataStreamUserRespresentation ("missing")]
+//    public class MissingPubSubDataViewStream : ViewStream
+//    {
+//       public MissingPubSubDataViewStream(string machine, string app, string kq, PubSubRoute[] routes, ViewStreamDatabase vsd)
+//       {
+//          ViewStream vs = new PubSubServerSubscriptionInfoStream(machine, app, kq, routes, vsd);
+//          vs = new FilterStream(vs, "app<>'' and publishers='' and deadPublishers=''");
+//          mySetRealSource(vs);
+//       }
+
+//       public MissingPubSubDataViewStream(string machine, string app, string kq, PubSubRoute[] routes)
+//          : this(machine, app, kq, routes, ViewStreamDatabase.instance)
+//       {
+//       }
+
+//       public MissingPubSubDataViewStream(string machine, string app, string kq, ViewStreamDatabase vsd)
+//          : this(machine, app, kq, PubSubServerSubscriptionInfoStream.defaultRoutes, vsd)
+//       {
+//       }
+
+//       public MissingPubSubDataViewStream(string machine, string app, string kq)
+//          : this(machine, app, kq, PubSubServerSubscriptionInfoStream.defaultRoutes, ViewStreamDatabase.instance)
+//       {
+//       }
+
+//       public MissingPubSubDataViewStream()
+//          : this(PubSubServerSubscriptionInfoStream.wildCard, 
+//                   PubSubServerSubscriptionInfoStream.wildCard, PubSubServerSubscriptionInfoStream.wildCard)
+//       {
+//       }
+//    }
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BCG.Utilities.General;
+using System.Collections;
+using System.IO;
+
+namespace BCG {
+   /// <summary>
+   /// Gets the subscription and publication state for one
+   /// server.
+   /// </summary>
+   public class PubSubServerStateStream : ViewStream, IWorker {
+      private static Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>> _stream_map =
+         new Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>>();
+      private KeysBasedTable2 _table =
+         new KeysBasedTable2(new BlueTableSchema("tagInfo",
+                                            new ColumnInfo("id", 0, typeof(int)),
+                                            new ColumnInfo("tag", 1, typeof(string)),
+                                            new ColumnInfo("server", 2, typeof(string)),
+                                            new ColumnInfo("subscribers", 3, typeof(string)),
+                                            new ColumnInfo("publishers", 4, typeof(string)),
+                                            new ColumnInfo("deadPublishers", 5, typeof(string))),
+                              new string[] { "tag" });
+      private SubscriberCore _sub_core;
+      private ViewStreamDatabase _vsd;
+
+      private static ViewStream _get_pub_sub_server_state_stream(ViewStreamDatabase vsd, string server_url) {
+         if (!_stream_map.ContainsKey(vsd))
+            _stream_map.Add(vsd, new Dictionary<string, ViewStream>());
+         if (!_stream_map[vsd].ContainsKey(server_url))
+            _stream_map[vsd].Add(server_url, new PubSubServerStateStream(server_url, vsd));
+         return _stream_map[vsd][server_url];
+      }
+
+
+      // public static ViewStream getPubSubServerStateStream(ViewStreamDatabase vsd, string server_uRL)
+      // {
+      //    return getPubSubServerStateStream(vsd, server_uRL);
+      // }
+
+
+      public PubSubServerStateStream(string server_url)
+     : this(server_url, ViewStreamDatabase.instance) { }
+
+      public PubSubServerStateStream(string server_url, ViewStreamDatabase vsd) {
+         _sub_core = new SubscriberCore(new PubSubRouter(new PubSubRoute(server_url, ".*")));
+         _vsd = vsd;
+      }
+
+
+      public PubSubServerStateStream()
+         : this(ViewStreamDatabase.instance) { }
+
+      public PubSubServerStateStream(ViewStreamDatabase vsd) {
+         mySetRealSource(new PubSubServerStateStream(vsd, PubSub3Config.instance.getSubRouting()));
+      }
+
+      public PubSubServerStateStream(ViewStreamDatabase vsd, PubSubRoute[] routes) {
+         List<ViewStream> vss = new List<ViewStream>();
+         foreach (var r in routes)
+            vss.Add(_get_pub_sub_server_state_stream(vsd, r.serverAddress));
+         mySetRealSource(new AppendStream(vss.ToArray()));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _sub_core.subscribe("System_AllSubscriptions");
+         _vsd.addWorker(this);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _sub_core.unsubscribe("System_AllSubscriptions");
+         _vsd.removeWorker(this);
+      }
+
+      #region IWorker Members
+
+      public void doWork() {
+         var updates = _sub_core.getUpdates(0);
+         // this is where the pubsub updates are converted to rows
+         if (updates.Count > 0) {
+            int id = 0;
+            BlueTable addedRows = new BlueTable(_table.schema);
+            foreach (PubSubUpdate u in updates) {
+               BinaryReader br = new BinaryReader(u.body);
+               int nUpdates = br.ReadInt32();
+               for (int i = 0; i < nUpdates; i++) {
+                  TagPubSubInfo tsi = new TagPubSubInfo(u.body);
+                  addedRows.addRow(new BlueRow(_table.schema, id++, tsi.tag, tsi.server, tsi.subscribersInString, tsi.publishersInString, tsi.deadPublishersInString));
+               }
+            }
+            IBlueRowCollection dRows, aRows;
+            _table.updateMultipleRows(addedRows, out dRows, out aRows);
+            myFireEvent(new DataPacket(_table, aRows, dRows));
+         }
+      }
+
+      #endregion
+
+      // public static string convertURLToTag(string URL) {
+      //    string[] urlWords = URL.Split('.');
+      //    string[] lastPart = urlWords[3].Split(':');
+      //    //the first one has "bluto://" in front of it
+      //    //we want to chop that off
+      //    string result = "bluto_" + urlWords[0].Substring(8) + "_" + urlWords[1] + "_" + urlWords[2] + "_" + lastPart[0] + "_" + lastPart[1];
+
+      //    return result;
+      // }
+   }
+}

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerSubscriptionInfoStream.cs

@@ -0,0 +1,174 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using BCG.Utilities.General;
+// using System.Collections;
+// using System.IO;
+
+// namespace BCG
+// {
+//    [DataStreamUserRespresentation("subInfo")]
+//    public class PubSubServerSubscriptionInfoStream : ViewStream
+//    {
+//       internal static string wildCard = ".*";
+//       internal static PubSubRoute[] defaultRoutes = PubSub3Config.instance.getSubRouting();
+//       private ViewStream myBaseStream;
+//       private IBlueTableSchema mySchema = new BlueTableSchema("SubInfo",
+//                                                                new ColumnInfo("id", 0, typeof(int)),
+//                                                                new ColumnInfo("machine", 1, typeof(string)),
+//                                                                new ColumnInfo("app", 2, typeof(string)),
+//                                                                new ColumnInfo("kq", 3, typeof(string)),
+//                                                                new ColumnInfo("pid", 4, typeof(int)),
+//                                                                new ColumnInfo("tag", 5, typeof(string)),
+//                                                                new ColumnInfo("connection", 6, typeof(string)),
+//                                                                new ColumnInfo("publishers", 7, typeof(string)),
+//                                                                new ColumnInfo("deadPublishers", 8, typeof(string)),
+//                                                                new ColumnInfo("server", 9, typeof(string)));
+
+//       private Hashtable myServerStateIdToSubInfoIdMap;
+//       private BlueTable myOutputRows;
+//       private int myIdToUse;
+//       public PubSubServerSubscriptionInfoStream(string serverURL, ViewStreamDatabase vsd)
+//       {
+
+//          myBaseStream = PubSubServerStateStream.getPubSubServerStateStream(vsd, serverURL);
+//          myServerStateIdToSubInfoIdMap = new Hashtable();
+//          myOutputRows = new BlueTable(mySchema);
+//          myIdToUse = 0;
+//       }
+
+//       public PubSubServerSubscriptionInfoStream(string machine, string app, string kq, PubSubRoute[] routes, ViewStreamDatabase vsd)
+//       {
+//          // if (PubSub3Config.instance.PS3_UseSubstations)
+//          // {
+//          //    //if we're using substations, we want to subscribe to each of these things by their service names. 
+//          //    //That means we ignore the routes coming in and just subscribe to all of the pubsub services advertised in bluepages
+//          //    SubstationServerBaseStream theRealSource = new SubstationServerBaseStream( (service,url) =>
+//          //       {
+//          //          ViewStream vs = new PubSubServerSubscriptionInfoStream(url, vsd);
+//          //             vs = new FilterStream(vs, new RegExpFilter(new string[] { "machine", "app", "kq" },
+//          //                                                   new string[] { machine, app, kq }));
+//          //             return vs;
+//          //       }
+//          //    );
+
+//          //    theRealSource.addAllSubstations(false);
+//          //    mySetRealSource(new ThreadSafeStream(vsd, new FilterStream(theRealSource)));
+
+
+//          // }
+//          //         else
+//          {
+//             List<ViewStream> vss = new List<ViewStream>();
+//             foreach (string url in PubSub3Config.instance.getAllPubSubs())
+//             {
+//                ViewStream vs = new PubSubServerSubscriptionInfoStream(url, vsd);
+//                vs = new FilterStream(vs, new RegExpFilter(new string[] { "machine", "app", "kq" },
+//                                                            new string[] { machine, app, kq }));
+//                vss.Add(vs);
+//             }
+//             mySetRealSource(new AppendStream(vss.ToArray()));
+//          }
+//       }
+
+//       public PubSubServerSubscriptionInfoStream(string machine, string app, string kq, PubSubRoute[] routes)
+//          : this(machine, app, kq, routes, ViewStreamDatabase.instance)
+//       { }
+
+//       public PubSubServerSubscriptionInfoStream(string machine, string app, string kq, ViewStreamDatabase vsd)
+//          : this(machine, app, kq, defaultRoutes, vsd)
+//       { }
+
+//       public PubSubServerSubscriptionInfoStream(string machine, string app, string kq)
+//          : this(machine, app, kq, defaultRoutes, ViewStreamDatabase.instance)
+//       { }
+
+//       public PubSubServerSubscriptionInfoStream()
+//          : this(wildCard, wildCard, wildCard)
+//       { }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myBaseStream.onChange += new ChangeDelegate(myHandleCallbacks);
+//       }
+
+//       void myHandleCallbacks(object newValue)
+//       {
+//          DataPacket dp = newValue as DataPacket;
+//          BlueTable outDeletedRows = new BlueTable(mySchema);
+//          BlueTable outAddedRows = new BlueTable(mySchema);
+
+//          foreach (IBlueRow dr in dp.deletedRows)
+//          {
+//             //find out which rows in the output table  correspond to the deleted row
+//             int[] outputRows = (int[])myServerStateIdToSubInfoIdMap[(int)dr["id"]];
+//             //if this thing wasn't deleted, just ignore it
+//             if (outputRows == null)
+//                continue;
+//             //delete all of those rows from the output table
+//             foreach (int deletedId in outputRows)
+//             {
+//                IBlueRow thisRow = myOutputRows[deletedId];
+//                myOutputRows.deleteRow(thisRow);
+//                outDeletedRows.addRow(thisRow);
+
+//             }
+
+//          }
+//          foreach (IBlueRow nr in dp.addedRows)
+//          {
+
+//             List<IBlueRow> outputRows = myTransFormToRows(nr);
+//             int[] outputRowIds = new int[outputRows.Count];
+//             int i = 0;
+//             foreach (IBlueRow r in outputRows)
+//             {
+//                myOutputRows.addRow(r);
+//                outputRowIds[i] = (int)r["id"];
+//                outAddedRows.addRow(r);
+//                ++i;
+//             }
+//             myServerStateIdToSubInfoIdMap[(int)nr["id"]] = outputRowIds;
+//          }
+//          if (myLastValue == null)
+//             myFireEvent(new DataPacket(myOutputRows, outAddedRows, outDeletedRows));
+//          else
+//             myFireEvent(new DataPacket(myOutputRows, outAddedRows, outDeletedRows));
+//       }
+
+//       private List<IBlueRow> myTransFormToRows(IBlueRow r)
+//       {
+//          List<IBlueRow> rv = new List<IBlueRow>();
+//          string tag = r["tag"].ToString();
+//          string server = r["server"].ToString();
+//          string publishers = r["publishers"].ToString();
+//          string deadPublishers = r["deadPublishers"].ToString();
+//          string[] subscribers = r["subscribers"].ToString().Split(',');
+//          if (r["subscribers"].ToString().Trim() != "")
+//          {
+//             foreach (string sub in subscribers)
+//             {
+//                string[] toks = sub.Split(';');
+//                IBlueRow nr = new BlueRow(mySchema, myIdToUse++, toks[0], toks[1], toks[2], Convert.ToInt64(toks[3]),
+//                                                       tag, toks[4], publishers, deadPublishers, server);
+//                rv.Add(nr);
+//             }
+//          }
+//          else
+//          {
+//             //publisher but no subscriber
+//             IBlueRow nr = new BlueRow(mySchema, myIdToUse++, "", "", "", DBNull.Value,
+//                                                    tag, "", publishers, deadPublishers, server);
+//             rv.Add(nr);
+//          }
+//          return rv;
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          myBaseStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+//       }
+
+//    }
+
+// }

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubTagStream.cs

@@ -0,0 +1,76 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Collections;
+
+namespace BCG
+{
+   /// <summary>
+   /// Gives the folowing information about every tag in the system :
+   ///  - last update
+   ///  - size of last update
+   /// This can be useful for watching non stream based tag, for which
+   /// theres no GUI. However, the stream subscribes to that tag, so
+   /// the subscription cost is there
+   /// </summary>
+   [DataStreamUserRespresentation ("tag")]
+   public class PubsubTagStream : ViewStream
+   {
+      private string[] myTags;
+      private ManagedSubscriber mySub;
+      private BlueTable myRows = new BlueTable (new BlueTableSchema ("Pubsubtags", 
+         new ColumnInfo ("id", 0, typeof(int)),
+         new ColumnInfo ("tag", 1, typeof(string)),
+         new ColumnInfo ("size", 2, typeof(int)),
+         new ColumnInfo ("lastUpdate", 3, typeof(DateTime))));
+      private IDictionary myTagRows = new Hashtable ();
+      private int myID = 0;
+
+      public PubsubTagStream(string tags)
+         : this(ViewStreamDatabase.instance, tags)
+      {}
+
+      public PubsubTagStream(ViewStreamDatabase vsd, string tags)
+      {
+         myTags = tags.Split(',');
+         mySub = ManagedSubscriber.instance;
+         vsd.addWorker(mySub);
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         foreach (string t in myTags)
+            mySub.subscribe(t.Trim(), new RawDataHandler(myRawDataHandler));
+      }
+
+      protected override void myOnLastSubscriber()
+      {
+         foreach (string t in myTags)
+            mySub.unsubscribe(t.Trim(), new RawDataHandler(myRawDataHandler));
+      }
+
+      private void myRawDataHandler(string tag, Stream s)
+      {
+         BlueTable dels = new BlueTable (myRows.schema),
+                     adds = new BlueTable (myRows.schema);
+         if (!myTagRows.Contains (tag))
+         {
+            MutableBlueRow r = new MutableBlueRow (myRows.schema, myID++, tag, s.Length, DateTime.Now);
+            myRows.addRow (r);
+            myTagRows.Add (tag, r);
+            adds.addRow (r);
+            myFireEvent (new DataPacket (myRows, adds, dels));
+         }
+         else
+         {
+            MutableBlueRow r = myTagRows[tag] as MutableBlueRow;
+            IBlueRow delRow = new BlueRow (r);
+            r.setValue ("size", s.Length);
+            r.setValue ("lastUpdate", DateTime.Now);
+            dels.addRow (delRow);
+            adds.addRow (r);
+            myFireEvent (new DataPacket (myRows, adds, dels));
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BCG.Utilities.General;
+using System.Collections;
+using System.IO;
+
+namespace BCG
+{
+   [DataStreamUserRespresentation("tagSynopsis")]
+   public class TagSynopsisStream : ViewStream, IStreamMapper, IWorker
+   {
+      private List<IWorker> myWorkers = new List<IWorker>();
+      IBlueTableSchema mySchema = new BlueTableSchema("AllTags",
+                                             new ColumnInfo("id", 0, typeof(int)),
+                                             new ColumnInfo("tag", 1, typeof(string)),
+                                             new ColumnInfo("subscribers", 2, typeof(int)),
+                                             new ColumnInfo("pubSub", 3, typeof(string)),
+                                             new ColumnInfo("dataPresent", 4, typeof(bool)));
+
+      public TagSynopsisStream()
+      {
+         mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.getRouter().getAllPubSubSubURLs()));
+      }
+
+      public TagSynopsisStream(string tagRegExp)
+      {
+         mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.getRouter().getAllPubSubSubURLs(), tagRegExp));
+      }
+
+      public TagSynopsisStream(ViewStreamDatabase vsd, string route)
+      {
+
+         ViewStream vs = new PubSubServerStateStream(route, vsd);
+         vs = new MapStream(vs, this);
+         myWorkers.Add(vs as IWorker);
+
+         mySetRealSource(vs);
+      }
+
+
+      public TagSynopsisStream(bool cons_overloader, string route) :
+      this(ViewStreamDatabase.instance, new string[] { route })
+      {
+      }
+
+      public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes)
+      {
+         List<ViewStream> vss = new List<ViewStream>();
+         foreach (string r in routes)
+         {
+            ViewStream vs = new PubSubServerStateStream(r, vsd);
+            vs = new MapStream(vs, this);
+            vss.Add(vs);
+            myWorkers.Add(vs as IWorker);
+         }
+         mySetRealSource(new AppendStream(vss.ToArray()));
+      }
+
+      public TagSynopsisStream(ViewStreamDatabase vsd, string tagRegExp, bool filterOutNonPublished)
+      {
+         // if (PubSub3Config.instance.PS3_UseSubstations)
+         // {
+         //    mySetRealSource(new ThreadSafeStream(new SubstationTagSynopsisStream(vsd, tagRegExp, filterOutNonPublished)));
+
+         // }
+         // else
+         {
+            PubSubRouter theRouter = new PubSubRouter(PubSub3Config.instance.getSubRouting());
+            ICollection<string> thisVSDRoutes = theRouter.getAllPubSubSubURLs();
+
+            ViewStream vs = new FilterStream(new TagSynopsisStream(vsd, thisVSDRoutes), "tag", tagRegExp);
+
+            if (filterOutNonPublished)
+               vs = new FilterStream(vs, "dataPresent=true");
+            mySetRealSource(vs);
+         }
+
+      }
+
+      public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes, string tagRegExp)
+      {
+         mySetRealSource(new FilterStream(new TagSynopsisStream(vsd, routes), "tag", tagRegExp));
+      }
+
+      public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes, string tagRegExp, bool filterOutNonPublished)
+      {
+         ViewStream vs = new TagSynopsisStream(vsd, routes, tagRegExp);
+         if (filterOutNonPublished)
+            vs = new FilterStream(vs, "dataPresent=true");
+         mySetRealSource(vs);
+      }
+
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         string subValue = row["subscribers"].ToString().Trim();
+         return new BlueRow(getOutputSchema(null),
+                           row["id"],
+                           row["tag"],
+                           subValue.Length == 0 ? 0 : subValue.Split(',').Length,
+                           row["server"],
+                           row["publishers"].ToString().Length > 0 || row["deadPublishers"].ToString().Length > 0);
+      }
+
+      #endregion
+
+      #region IWorker Members
+
+      public void doWork()
+      {
+         foreach (IWorker w in myWorkers)
+            w.doWork();
+      }
+
+      #endregion
+   }
+}

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -19,28 +19,35 @@ using Spark.Core.Protobuf;
 using mbts.ui;
 // using mbts.bb;
 
-namespace mbts.spark {
-   internal class ResourceAllocator : IDisposable {
+namespace mbts.spark
+{
+   internal class ResourceAllocator : IDisposable
+   {
       private List<IDisposable> _disps = new List<IDisposable>();
       private Dictionary<SecurityTuple, SingleOptionDriverPricePublisher> _option_driver_map =
          new Dictionary<SecurityTuple, SingleOptionDriverPricePublisher>();
       ThreadedBlackboard _bb;
-      public ResourceAllocator(IBus bus, ThreadedBlackboard bb) {
+      public ResourceAllocator(IBus bus, ThreadedBlackboard bb)
+      {
          bts.Log.Gen.info("ResourceAllocator started");
          _bb = bb;
          var s = bus.subscribe(SparkTags.SystemContractsTag).OnDispatcher().Subscribe(_on_data);
          _disps.Add(s);
       }
-      private void _on_data(SystemContractData d) {
+      private void _on_data(SystemContractData d)
+      {
          bts.Log.Gen.info(String.Format("ResourceAllocator._on_data sz: {0}", d.sec_md_map.Count()));
-         foreach (SecurityTuple sec_tup in d.get_options()) {
-            if (!_option_driver_map.ContainsKey(sec_tup)) {
+         foreach (SecurityTuple sec_tup in d.get_options())
+         {
+            if (!_option_driver_map.ContainsKey(sec_tup))
+            {
                bts.Log.Gen.info($"ResourceAllocator._on_data allocating SingleOptionDriverPricePublisher for '{sec_tup}'");
                _option_driver_map.Add(sec_tup, new SingleOptionDriverPricePublisher(sec_tup, _bb));
             }
          }
       }
-      public void Dispose() {
+      public void Dispose()
+      {
          _disps.ForEach(d => d.Dispose());
       }
    }
@@ -54,7 +61,8 @@ namespace mbts.spark {
    //  - Activation Tag for newly created VS's (VolSurfaceActivationTag) (retired)
    // 2. Listens to the activation tag (also published by the UI) and
    //    creates the SingleVolSurfaceManager for the VS   
-   public class VolSurfaceBusManager : IDisposable {
+   public class VolSurfaceBusManager : IDisposable
+   {
       private List<IDisposable> _disps = new List<IDisposable>();
       private ThreadedBlackboard _bb;
       private bts.api.analytics.AnalyticsService _analytics_service;
@@ -78,7 +86,8 @@ namespace mbts.spark {
 
       public VolSurfaceBusManager(bts.api.IBus bus,
                                          bts.api.analytics.AnalyticsService analytics_service,
-                                         bts.api.orderman.OrderService order_service) {
+                                         bts.api.orderman.OrderService order_service)
+      {
          bts.Log.Ui.info("VSBusManager cons tid:{VolSurfaceTroubleshootingFunctions.tid()}");
          _bus = bus;
          _bb = new ThreadedBlackboard("AW2", bus);
@@ -115,29 +124,37 @@ namespace mbts.spark {
          _subs.Add(new InterpTheosServerPublisher(_bb, order_service));
          _subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
          // usage         
-         var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t => {
+         var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
+         {
             VolSurfaceUsageList ulist = new VolSurfaceUsageList();
             IDictionary<string, int> name_to_ids = new Dictionary<string, int>();
-            foreach (var kv in t) {
+            foreach (var kv in t)
+            {
                name_to_ids[kv.Value] = kv.Key;
             }
             Log.Ui.info($"VolSurfaceUsageList name_to_ids count: {name_to_ids.Count}");
             var vs_usage = mbts.config.Configuration.get<bts.config.toggles.VolSurfaceUsage>();
             Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");
-            foreach (var u in vs_usage.Items) {
+            foreach (var u in vs_usage.Items)
+            {
                string option_name = u.OptionName;
                string vs_name = u.VsName;
                int vs_key;
-               if (name_to_ids.TryGetValue(vs_name, out vs_key)) {
-                  ulist.UsageList.Add(new VolSurfaceUsageEntry() {
+               if (name_to_ids.TryGetValue(vs_name, out vs_key))
+               {
+                  ulist.UsageList.Add(new VolSurfaceUsageEntry()
+                  {
                      VsId = vs_key,
-                     SecTup = new bts.api.SecurityTuple {
+                     SecTup = new bts.api.SecurityTuple
+                     {
                         Type = SecurityType.Option,
                         Venue = Venue.Deribit,
                         Symbol = option_name
                      }
                   });
-               } else {
+               }
+               else
+               {
                   Log.Gen.error($"Could not find vol surface with name: {vs_name}");
                }
             }
@@ -153,7 +170,8 @@ namespace mbts.spark {
          _bus.subscribe(SparkTags.TC2Tags.AllLiveTheosTag)
             .CombineLatest(bus.subscribe(SparkTags.SystemContractsTag),
             (theos, sc_data) => (theos, sc_data)).Subscribe(
-               tup => {
+               tup =>
+               {
                   var keyed_theos = _assemble_keyed_theos(tup.sc_data, tup.theos);
                   _bus.post(SparkTags.KeyedTheos, keyed_theos);
                }
@@ -163,33 +181,40 @@ namespace mbts.spark {
 
       static IReadOnlyDictionary<uint, VolSurfaceTheoPoint> _assemble_keyed_theos(
          SystemContractData sc_data,
-         IReadOnlyDictionary<string, ExpStrikeKeyedData<TheoPoint>> theos) {
-         if (sc_data is null || theos is null) {
+         IReadOnlyDictionary<string, ExpStrikeKeyedData<TheoPoint>> theos)
+      {
+         if (sc_data is null || theos is null)
+         {
             return new Dictionary<uint, VolSurfaceTheoPoint>(0);
          }
 
          HighPerformanceTimer timer = new HighPerformanceTimer();
          timer.start();
          Dictionary<uint, VolSurfaceTheoPoint> rv = new Dictionary<uint, VolSurfaceTheoPoint>();
-         foreach (var kv in theos) {
+         foreach (var kv in theos)
+         {
             var symbol = kv.Key;
             ExpStrikeKeyedData<TheoPoint> sym_theos = kv.Value;
             if (sym_theos == null)
                continue;
-            foreach (var s_kv in sym_theos.get_data_linearly()) {
+            foreach (var s_kv in sym_theos.get_data_linearly())
+            {
                DateTime exp = s_kv.first;
                double strike = s_kv.second;
                TheoPoint tp = s_kv.third;
                // TODO: fix exchange later
                Tuple<uint?, uint?> keys =
                   sc_data.get_option_sec_id(Venue.Deribit, symbol, exp, strike);
-               if (keys != null) {
+               if (keys != null)
+               {
                   var c_key = keys.Item1;
                   var p_key = keys.Item2;
-                  if (c_key.HasValue) {
+                  if (c_key.HasValue)
+                  {
                      rv[c_key.Value] = new VolSurfaceTheoPoint(tp, true);
                   }
-                  if (p_key.HasValue) {
+                  if (p_key.HasValue)
+                  {
                      rv[p_key.Value] = new VolSurfaceTheoPoint(tp, false);
                   }
                }
@@ -201,78 +226,100 @@ namespace mbts.spark {
          return rv;
       }
 
-      public ThreadedBlackboard bb() {
+      public ThreadedBlackboard bb()
+      {
          return _bb;
       }
 
-      void _spin_vss(VolSurfaceUsageListImmutable ulist) {
-         foreach (var u in ulist.UsageList) {
+      void _spin_vss(VolSurfaceUsageListImmutable ulist)
+      {
+         foreach (var u in ulist.UsageList)
+         {
             var vs_id = u.VsId;
             _bb.post(SparkTags.VolSurfaceActiveUITag(vs_id), true);
          }
       }
 
-      void _on_vs_usage_update(Update<VolSurfaceUsageListImmutable, int> update) {
+      void _on_vs_usage_update(Update<VolSurfaceUsageListImmutable, int> update)
+      {
          // TODO
          return;
       }
 
-      void _on_vss(bts.api.core.Update<VolSurfaceEntryImmutable, UInt64> update) {
+      void _on_vss(bts.api.core.Update<VolSurfaceEntryImmutable, UInt64> update)
+      {
          bts.Log.Gen.info(String.Format("VSBusManager._on_vss tid:{0} count:{1}",
                           System.Threading.Thread.CurrentThread.ManagedThreadId, update.all.Count()));
          IDictionary<int, string> to_publish = new Dictionary<int, string>();
          // publish a dictionary of vs id vs vs names
-         foreach (VolSurfaceEntryImmutable vs in update.all) {
+         foreach (VolSurfaceEntryImmutable vs in update.all)
+         {
             to_publish[vs.VsId] = vs.VsName;
          }
          _bb.post(SparkTags.VolSurfaceListTag, to_publish);
          _init_vss(to_publish.Keys);
       }
 
-      private void _init_vss(IEnumerable<int> vs_ids) {
+      private void _init_vss(IEnumerable<int> vs_ids)
+      {
          // we consider every vs in the db as "activated" which might prove to be expensive
          // (will take care of it later)
-         foreach (var vs_id in vs_ids) {
-            if (!_vs_manager_map.ContainsKey(vs_id)) {
+         foreach (var vs_id in vs_ids)
+         {
+            if (!_vs_manager_map.ContainsKey(vs_id))
+            {
                _vs_manager_map.Add(vs_id, new SingleVolSurfaceManager(
                   vs_id, _bb, _analytics_service));
             }
          }
       }
 
-      private void _on_bb_update(object tag, object data) {
-         if (_in_scdata_tag.Equals(tag)) {
+      private void _on_bb_update(object tag, object data)
+      {
+         if (_in_scdata_tag.Equals(tag))
+         {
             _contract_data = _bb.read_state(_in_scdata_tag);
          }
-         if (_in_bb_create_tag.Equals(tag)) {
+         if (_in_bb_create_tag.Equals(tag))
+         {
             // creations are written in a queue and handled later in the timer
             // callback
             mbts.analytics.VolSurfaceSettings vs_settings = data as mbts.analytics.VolSurfaceSettings;
             bts.Log.Gen.info(String.Format("VSBusManager._on_bb_update received vs creation tag vs:{0}",
                               ProtobufSerializationHelper.serialize_to_string(vs_settings)));
             _waiting_creations.Enqueue(vs_settings);
-         } else if (_in_bb_delete_tag.Equals(tag)) {
+         }
+         else if (_in_bb_delete_tag.Equals(tag))
+         {
             int vs_id = (int)data;
-            try {
+            try
+            {
                IntWrapper vs_id_w = new IntWrapper();
                vs_id_w.Data = vs_id;
                _analytics_service.delete_volsurface(vs_id_w);
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                Log.Gen.error(String.Format("Vol Surface deletion failed with exception:{0}", ex.Message));
                _bb.post(SparkTags.MessageDisplayTag,
                         new MessageDisplayData("Vol Surface Deletion",
                                  String.Format("Could not delete Vol Surface, Error : {0}", ex.Message)));
             }
-         } else if (_in_bb_exp_params_tag.Equals(tag)) {
+         }
+         else if (_in_bb_exp_params_tag.Equals(tag))
+         {
             AnalyticsExpirationParametersMap exp_param_map = data as AnalyticsExpirationParametersMap;
             // TODO, add whatever else is needed to AnalyticsExpirationParametersMap -> VolSurfaceNonVolInputTag
             _bb.post(SparkTags.VolSurfaceNonVolInputTag,
                                           new mbts.analytics.VolSurfaceNonVolInput(exp_param_map.exp_map.ToDictionary()));
-         } else if (_in_bb_req_vol.Equals(tag)) {
+         }
+         else if (_in_bb_req_vol.Equals(tag))
+         {
             ClientArchivedVolRequest client_req = _bb.read_state(_in_bb_req_vol);
             bts.Log.Ui.info($"VolSurfaceBusManager._on_bb_update ClientArchivedVolRequest: {client_req.ToString()}");
             ArchivedTheoRetrieval ret = client_req.ret;
-            if (ret == null || ret.SecTup == null) {
+            if (ret == null || ret.SecTup == null)
+            {
                bts.Log.Ui.warn($"VolSurfaceBusManager malformed ArchivedTheoRetrieval");
                ArchivedTheoList empty = new ArchivedTheoList();
                _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, empty));
@@ -290,25 +337,30 @@ namespace mbts.spark {
       }
 
       private int _timer_count = 0;
-      private void _on_timer() {
-         Action<string> report_err = (string err_txt) => {
+      private void _on_timer()
+      {
+         Action<string> report_err = (string err_txt) =>
+         {
             Log.Gen.error(String.Format("VSBusManager, creation failed with message:{0}", err_txt));
             _bb.post(SparkTags.MessageDisplayTag, new MessageDisplayData("Vol Surface Addition",
                                           String.Format("Could not create Vol Surface, Error : {0}", err_txt)));
          };
          bts.Log.Gen.diag(String.Format("VSBusManager._on_timer tid:{0} count:{1}",
                          System.Threading.Thread.CurrentThread.ManagedThreadId, _timer_count++));
-         if (_waiting_creations.Count == 0) {
+         if (_waiting_creations.Count == 0)
+         {
             return;
          }
-         if (_contract_data == null) {
+         if (_contract_data == null)
+         {
             return;
          }
          mbts.analytics.VolSurfaceSettings vs_settings = _waiting_creations.Dequeue();
          IDictionary<string, IEnumerable<DateTime>> exp_map = _contract_data.get_option_expirations(VolSurfaceDataFunctions.get_symbols(vs_settings), false);
          VolSurfaceInput vs_input = mbts.analytics.VolSurfaceDataFunctions.get_default_vs_input(vs_settings, exp_map);
          var reply = _analytics_service.get_new_volsurface_id(new VoidMessage());
-         if (!reply.Ok) {
+         if (!reply.Ok)
+         {
             report_err("Could not obtain new ID");
             return;
          }
@@ -317,7 +369,8 @@ namespace mbts.spark {
          _timer.start();
          ResultMessage res = VolSurfaceUtilityFunctions.save_volsurface(vs_input, _analytics_service);
          _timer.stop();
-         if (!res.Success) {
+         if (!res.Success)
+         {
             report_err(res.ErrorMessage);
             return;
          }
@@ -327,7 +380,8 @@ namespace mbts.spark {
          _bb.post(SparkTags.VolSurfaceActivationTag(reply.NewId), true, false);
       }
 
-      public void Dispose() {
+      public void Dispose()
+      {
          _subs.ForEach(s => s.Dispose());
       }
 

src/client/spark/AW2/Misc/Pair.cs

@@ -1,78 +1,78 @@
-using ProtoBuf;
+// using ProtoBuf;
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Windows;
-using System.Windows.Media;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.Threading.Tasks;
+// using System.Windows;
+// using System.Windows.Media;
 
-namespace mbts.spark {
-   // TODO move it out or replace it with Tuple
-   [Serializable]
-   public class Pair<F, S> {
-      public F first;
-      public S second;
+// namespace mbts.spark {
+//    // TODO move it out or replace it with Tuple
+//    [Serializable]
+//    public class Pair<F, S> {
+//       public F first;
+//       public S second;
 
-      public Pair() { }
-      public Pair(F f, S s) {
-         first = f;
-         second = s;
-      }
+//       public Pair() { }
+//       public Pair(F f, S s) {
+//          first = f;
+//          second = s;
+//       }
 
-      public Pair(KeyValuePair<F, S> kvp)
-         : this(kvp.Key, kvp.Value) { }
+//       public Pair(KeyValuePair<F, S> kvp)
+//          : this(kvp.Key, kvp.Value) { }
 
-      public override string ToString() {
-         return String.Format("{0},{1}", first != null ? first.ToString() : "NULL",
-                                 second != null ? second.ToString() : "NULL");
-      }
+//       public override string ToString() {
+//          return String.Format("{0},{1}", first != null ? first.ToString() : "NULL",
+//                                  second != null ? second.ToString() : "NULL");
+//       }
 
-      public override int GetHashCode() {
-         return ToString().GetHashCode();
-      }
+//       public override int GetHashCode() {
+//          return ToString().GetHashCode();
+//       }
 
-      public override bool Equals(object obj) {
-         return obj != null && obj is Pair<F, S> && ToString() == obj.ToString();
-      }
-   }
+//       public override bool Equals(object obj) {
+//          return obj != null && obj is Pair<F, S> && ToString() == obj.ToString();
+//       }
+//    }
 
-   [Serializable]
-   public class Triplet<F, S, T> {
-      public F first;
-      public S second;
-      public T third;
+//    [Serializable]
+//    public class Triplet<F, S, T> {
+//       public F first;
+//       public S second;
+//       public T third;
 
-      public Triplet() { }
+//       public Triplet() { }
 
-      public Triplet(Tuple<F, S, T> tp) {
-         first = tp.Item1;
-         second = tp.Item2;
-         third = tp.Item3;
-      }
+//       public Triplet(Tuple<F, S, T> tp) {
+//          first = tp.Item1;
+//          second = tp.Item2;
+//          third = tp.Item3;
+//       }
 
-      public Triplet(F f, S s, T t) {
-         first = f;
-         second = s;
-         third = t;
-      }
+//       public Triplet(F f, S s, T t) {
+//          first = f;
+//          second = s;
+//          third = t;
+//       }
 
-      public static Triplet<F, S, T> from_tuple(Tuple<F, S, T> t) { return new Triplet<F, S, T>(t.Item1, t.Item2, t.Item3); }
+//       public static Triplet<F, S, T> from_tuple(Tuple<F, S, T> t) { return new Triplet<F, S, T>(t.Item1, t.Item2, t.Item3); }
 
-      public override string ToString() {
-         return String.Format("{0},{1},{2}", first != null ? first.ToString() : "NULL",
-                                             second != null ? second.ToString() : "NULL",
-                                             third != null ? third.ToString() : "NULL");
-      }
+//       public override string ToString() {
+//          return String.Format("{0},{1},{2}", first != null ? first.ToString() : "NULL",
+//                                              second != null ? second.ToString() : "NULL",
+//                                              third != null ? third.ToString() : "NULL");
+//       }
 
-      public override bool Equals(object obj) {
-         return obj != null && obj is Triplet<F, S, T> && ToString() == obj.ToString();
-      }
+//       public override bool Equals(object obj) {
+//          return obj != null && obj is Triplet<F, S, T> && ToString() == obj.ToString();
+//       }
 
-      public override int GetHashCode() {
-         return ToString().GetHashCode();
-      }
-   }
+//       public override int GetHashCode() {
+//          return ToString().GetHashCode();
+//       }
+//    }
 
-}
+// }

src/client/spark/Analytics/MarketImpliedVol/VolSurfaceMarketFitSettings.cs

@@ -4,9 +4,11 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace mbts.analytics {
+namespace mbts.analytics
+{
    [ProtoBuf.ProtoContract(Name = "VolSurfaceMarketFitSettings")]
-   public class VolSurfaceMarketFitSettings {
+   public class VolSurfaceMarketFitSettings
+   {
       /// use one sided market or not
       [ProtoBuf.ProtoMember(10)]
       public bool ignore_one_sided_markets;
@@ -30,23 +32,27 @@ namespace mbts.analytics {
    }
 
    [ProtoBuf.ProtoContract(Name = "VolSurfaceAutoFitSettings")]
-   public class VolSurfaceAutoFitSettings {
+   public class VolSurfaceAutoFitSettings
+   {
       // each of these settings can be enabled independently. if they are enabled
       // then any of them being true will result in an autmatic fit
       [ProtoBuf.ProtoContract(Name = "TimeBasedSettings")]
-      public class TimeBasedSettings {
+      public class TimeBasedSettings
+      {
          [ProtoBuf.ProtoMember(10)]
          public uint? period_seconds { get; set; }
       }
 
       [ProtoBuf.ProtoContract(Name = "DriverMoveBasedSettings")]
-      public class DriverMoveBasedSettings {
+      public class DriverMoveBasedSettings
+      {
          [ProtoBuf.ProtoMember(10)]
          public double? points_moved { get; set; }
       }
 
       [ProtoBuf.ProtoContract(Name = "ATMImpliedVolBasedSettings")]
-      public class ATMImpliedVolBasedSettings {
+      public class ATMImpliedVolBasedSettings
+      {
          [ProtoBuf.ProtoMember(20)]
          public double? points_moved { get; set; }
          // we can do price terms moved as well

src/client/spark/Daemons/MergedMDDaemon.cs

@@ -23,9 +23,11 @@ using System.Threading.Tasks;
 
 using SecurityKey = System.UInt32;
 
-namespace bts {
+namespace bts
+{
    [Export(typeof(IShellDaemon))]
-   internal class MergedMDDaemon : IShellDaemon, IDisposable {
+   internal class MergedMDDaemon : IShellDaemon, IDisposable
+   {
       private readonly IBus _bus;
       IDisposable _dispose;
       Dictionary<uint, SecurityMD> _server_sec_map = new Dictionary<uint, SecurityMD>();
@@ -33,7 +35,8 @@ namespace bts {
       private readonly IList<IDisposable> _subs = new List<IDisposable>();
 
       [ImportingConstructor]
-      public MergedMDDaemon(IBus bus) {
+      public MergedMDDaemon(IBus bus)
+      {
          //TempTester.test_analytics();
          bts.Log.Gen.info("MergedMDDaemon started");
          _bus = bus;
@@ -46,17 +49,43 @@ namespace bts {
          // works with the various UI windows)
          // _subs.Add(_bus.subscribe(SparkTags.SparkTimer).BufferLatest(TimeSpan.FromMinutes(1))
          //             .Subscribe(t => _merge_and_post()));
+         //_test_md_stream();
+      }
+      // BCG.MDStream _md_stream = null;
+      // private void _test_md_stream()
+      // {
+      //    _bus.subscribe(SparkTags.SparkTimer).Subscribe(t =>
+      //       {
+      //          if (_md_stream == null)
+      //          {
+      //             _md_stream = new BCG.MDStream();
+      //             _md_stream.onChange += _on_md;
+      //          }
+      //          else
+      //          {
+      //             BCG.ViewStreamDatabase.instance.doWork();
+      //          }
+      //       });
+      // }
+
+      private void _on_md(object obj)
+      {
+         BCG.DataPacket dp = obj as DataPacket;
+         bts.Log.Gen.info($"MDStream callback dp: {dp}");
       }
 
-      private void _log_options(IReadOnlyDictionary<uint, SecurityMD> sec_map) {
+      private void _log_options(IReadOnlyDictionary<uint, SecurityMD> sec_map)
+      {
          string rv = "MergedMDDaemon Options:\n";
-         foreach (var kv in sec_map) {
+         foreach (var kv in sec_map)
+         {
             var sec = kv.Value.security;
             var md = kv.Value.md;
             ulong time = 0;
             NativePriceSize bid, ask;
             var has_md = md.get_bbo(ref time, out bid, out ask);
-            if (sec.HasOption) {
+            if (sec.HasOption)
+            {
                rv += $"id: {kv.Key} desc: {sec.security_desc()} sym: {sec.Symbol} " +
                      $"exp: {sec.security_expiration()} " +
                      $"k: {sec.Option.Strike} cp: {sec.Option.CallPut} " +
@@ -67,26 +96,31 @@ namespace bts {
          Log.Gen.info(rv);
       }
 
-      private void _on_server_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map) {
+      private void _on_server_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map)
+      {
          //_log_options(sec_map);
          bts.Log.Gen.info(String.Format("MergedMDDaemon._on_server_sec_map sz: {0}", sec_map.Count()));
          _server_sec_map = new Dictionary<uint, SecurityMD>(sec_map);
          _merge_and_post();
       }
 
-      private void _on_fake_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map) {
+      private void _on_fake_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map)
+      {
          bts.Log.Gen.info(String.Format("MergedMDDaemon._on_fake_sec_map sz: {0}", sec_map.Count()));
          _fake_sec_map = new Dictionary<uint, SecurityMD>(sec_map);
          _merge_and_post();
       }
 
-      private void _merge_and_post() {
-         Func<SecurityMD, bool> is_active = (sec_md) => {
+      private void _merge_and_post()
+      {
+         Func<SecurityMD, bool> is_active = (sec_md) =>
+         {
             var sec = sec_md.security;
             return SecurityExpirationHelper.is_unexpired(sec);
          };
          var to_pub = _server_sec_map.Where(kv => is_active(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value);
-         foreach (var kv in _fake_sec_map) {
+         foreach (var kv in _fake_sec_map)
+         {
             to_pub[kv.Key] = kv.Value;
          }
          bts.Log.Gen.info(String.Format("MergedMDDaemon._merge_and_post sz: {0}", to_pub.Count()));

src/client/spark/NMSheets/BlackboardStructure/MNSheetTheoRequest.cs

@@ -3,21 +3,24 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
+using mbts;
 using mbts.spark;
 
-namespace BCG {
+namespace BCG
+{
    /// This is the theo request for an NMSheetGrid. Its send out by 
    /// the driver stip at the top (NMSheetDriverControlStrip). The
    /// NMSheetSingleOptionMainDataSource filters it for its own
    /// symbol and computes the data.
-   public class NMSheetDataRequest {
+   public class NMSheetDataRequest
+   {
       public readonly IEnumerable<Triplet<NMSheetExpiration, bool, double>> expiration_request_list;
       public NMSheetDataRequest(IEnumerable<Triplet<NMSheetExpiration, bool, double>> the_expiration_request_list) { expiration_request_list = the_expiration_request_list; }
 
       /// Returns a new request thats restricted for the
       /// symbol in the argument
-      public static NMSheetDataRequest restrict_for_symbol(NMSheetDataRequest request, NMSheetSymbol option_symbol) {
+      public static NMSheetDataRequest restrict_for_symbol(NMSheetDataRequest request, NMSheetSymbol option_symbol)
+      {
          List<Triplet<NMSheetExpiration, bool, double>> rv = new List<Triplet<NMSheetExpiration, bool, double>>();
          foreach (Triplet<NMSheetExpiration, bool, double> t in request.expiration_request_list)
             if (t.first.option_name == option_symbol.option_name && t.first.option_exchange == option_symbol.option_exchange)
@@ -27,7 +30,8 @@ namespace BCG {
 
       /// Returns a new request thats restricted for the
       /// symbol in the argument
-      public static NMSheetDataRequest restrict_for_expirations(NMSheetDataRequest request, IEnumerable<NMSheetExpiration> exps) {
+      public static NMSheetDataRequest restrict_for_expirations(NMSheetDataRequest request, IEnumerable<NMSheetExpiration> exps)
+      {
          HashSet<NMSheetExpiration> exp_map = new HashSet<NMSheetExpiration>(exps);
          List<Triplet<NMSheetExpiration, bool, double>> rv = new List<Triplet<NMSheetExpiration, bool, double>>();
          foreach (Triplet<NMSheetExpiration, bool, double> t in request.expiration_request_list)

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionData.cs

@@ -6,7 +6,7 @@ using System.Text;
 using mbts.ui;
 using bts.utils;
 using mbts.spark;
-
+using mbts;
 namespace BCG {
    /// <summary>
    /// Holds together data for one NMSheetSymbol. Its constructed by

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionDataTaskEye.cs

@@ -1,16 +1,19 @@
 using System;
 
 using bts.api;
-
+using mbts;
 using mbts.spark;
 using mbts.ui;
 
-namespace BCG {
-   public partial class NMSheetSingleOptionData {
+namespace BCG
+{
+   public partial class NMSheetSingleOptionData
+   {
       // Gets the data from the bus, and fills in out_eye_data, and cp_row
       private void _fill_eye_data(Triplet<mbts.spark.TheoPoint, uint, uint> tp_with_seckeys,
                                   ref double[] cp_row,
-                                  out NMSheetCallPutBidAskWrapper<NMSheetTaskEyeData> out_eye_data) {
+                                  out NMSheetCallPutBidAskWrapper<NMSheetTaskEyeData> out_eye_data)
+      {
          NMSheetTaskEyeData call_bid_data, call_ask_data, put_bid_data, put_ask_data;
          // call
          _get_eye_security_side_data(tp_with_seckeys.second, Side.Buy, out call_bid_data);
@@ -23,7 +26,8 @@ namespace BCG {
       }
 
       private void _get_eye_security_side_data(uint sec_key, Side side,
-                                               out NMSheetTaskEyeData inp_out_data) {
+                                               out NMSheetTaskEyeData inp_out_data)
+      {
          inp_out_data = null;
          if (_ee_tasking_data == null)
             return;

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionDataTaskQuoter.cs

@@ -1,12 +1,15 @@
 using System;
 
 using bts.api;
-
+using mbts;
 using mbts.spark;
 
-namespace BCG {
-   public partial class NMSheetSingleOptionData {
-      private void _fill_quoter_data(Triplet<mbts.spark.TheoPoint, uint, uint> tp_with_seckeys, Tuple<double, double> driver_bid_ask, double[] cp_row, out NMSheetCallPutBidAskWrapper<NMSheetTaskQuoterData> out_quoter_data) {
+namespace BCG
+{
+   public partial class NMSheetSingleOptionData
+   {
+      private void _fill_quoter_data(Triplet<mbts.spark.TheoPoint, uint, uint> tp_with_seckeys, Tuple<double, double> driver_bid_ask, double[] cp_row, out NMSheetCallPutBidAskWrapper<NMSheetTaskQuoterData> out_quoter_data)
+      {
          NMSheetTaskQuoterData call_bid_data, call_ask_data, put_bid_data, put_ask_data;
          _get_quoter_security_side_data(tp_with_seckeys.first, PutOrCall.Call, Side.Buy, tp_with_seckeys.second, driver_bid_ask, out call_bid_data, ref cp_row,
                                         NMSheetColumnName.call_quoter_set_bid_edge, NMSheetColumnName.call_quoter_set_bid_size, NMSheetColumnName.call_quoter_bid_price, NMSheetColumnName.call_quoter_bid_size);
@@ -23,7 +26,8 @@ namespace BCG {
                                                    out NMSheetTaskQuoterData sheet_quoter_data,
                                                    ref double[] out_array,
                                                    NMSheetColumnName inp_edge_col, NMSheetColumnName inp_size_col,
-                                                   NMSheetColumnName quoter_price_col, NMSheetColumnName quoter_size_col) {
+                                                   NMSheetColumnName quoter_price_col, NMSheetColumnName quoter_size_col)
+      {
          out_array = new double[] { };
          sheet_quoter_data = new NMSheetTaskQuoterData(new mbts.tasks.TaskSecurityQuoterData());
          // mbts.tasks.TaskSecurityQuoterData quoter_data = _tasking_data.get_quoter_data(sec_key, side);

src/client/spark/NMSheets/DataProviders/NMSheetMainDataSource.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
-
+using mbts;
 using BCG.Utilities;
 using BCG.Utilities.General;
 

src/client/spark/NMSheets/Outside/ColorRetriever.cs

@@ -5,7 +5,7 @@ using System.Windows;
 using System.Windows.Controls;
 using System.Collections.Generic;
 using System.Windows.Media;
-
+using mbts;
 using mbts.ui;
 using mbts.spark;
 
@@ -34,7 +34,7 @@ namespace BCG
 
       public static System.Drawing.Color convert_color(BTSColor o_color)
       {
-         return (System.Drawing.Color) o_color;
+         return (System.Drawing.Color)o_color;
       }
 
       public static Color clone_color(Color color)

src/client/spark/NMSheets/Outside/ExchangeSecurityMap.cs

@@ -5,10 +5,11 @@ using System.Text;
 using System.Threading.Tasks;
 using mbts.ui;
 using bts.api;
-
+using mbts;
 using mbts.spark;
 
-namespace BCG {
+namespace BCG
+{
    /// <summary>
    /// This is the list of all exchange listed securities
    /// in the system. It provides :
@@ -17,7 +18,8 @@ namespace BCG {
    /// </summary>
    // TODO: SystemSecurityData will replace this (for backwards Tc2 bckwards compat
    // it is created from SystemContractData)
-   public class ExchangeSecurityMap/* : mbts.bb.IExchangeSecurityMap */{
+   public class ExchangeSecurityMap/* : mbts.bb.IExchangeSecurityMap */
+   {
       public const uint error_key = 0;
       private readonly IDictionary<uint, Security> _key_sec_map;
       private readonly IDictionary<mbts.SecurityStrike, Security[]> _option_map;
@@ -25,17 +27,22 @@ namespace BCG {
       private readonly IDictionary<Venue, IDictionary<string, Security>> _spread_map;
 
 
-      public ExchangeSecurityMap(SystemContractData sc_data) {
-         Func<UInt32, Security> get_sec = (id) => {
+      public ExchangeSecurityMap(SystemContractData sc_data)
+      {
+         Func<UInt32, Security> get_sec = (id) =>
+         {
             return sc_data.sec_md_map[id].security.CloneToMutable();
          };
          var key_sec_map_aux = new Dictionary<uint, Security>();
-         foreach (var kv in sc_data.sec_md_map) {
+         foreach (var kv in sc_data.sec_md_map)
+         {
             key_sec_map_aux[kv.Key] = kv.Value.security.CloneToMutable();
          }
          var option_map_aux = new Dictionary<mbts.SecurityStrike, Security[]>();
-         foreach (var o_kv in sc_data.strike_map) {
-            foreach (var k in o_kv.Value) {
+         foreach (var o_kv in sc_data.strike_map)
+         {
+            foreach (var k in o_kv.Value)
+            {
                var (call_id, put_id) = k.call_put_id;
                if (call_id.HasValue && put_id.HasValue)
                   option_map_aux[k] = new[] { get_sec(call_id.Value), get_sec(put_id.Value) };
@@ -52,7 +59,8 @@ namespace BCG {
       public ExchangeSecurityMap(IDictionary<uint, Security> key_sec_map,
                                  IDictionary<mbts.SecurityStrike, Security[]> option_map,
                                  IDictionary<mbts.SecurityExpiration, Security> fut_map,
-                                 IDictionary<Venue, IDictionary<string, Security>> spread_map) {
+                                 IDictionary<Venue, IDictionary<string, Security>> spread_map)
+      {
          _key_sec_map = key_sec_map;
          _option_map = option_map;
          _fut_map = fut_map;
@@ -61,19 +69,22 @@ namespace BCG {
 
       #region IExchangeSecurityMap
       public Security get_option_security(Venue exchange, string symbol, DateTime exp,
-                                           double strike, PutOrCall cp) {
+                                           double strike, PutOrCall cp)
+      {
          mbts.SecurityStrike k = new mbts.SecurityStrike(new mbts.SecurityExpiration(exchange, SecurityType.Option, symbol, exp), strike);
          Security[] rv = _option_map.TryGetValue(k, out rv) ? rv : null;
          return rv != null ? rv[cp == PutOrCall.Call ? 0 : 1] : null;
       }
 
-      public Security get_future_security(Venue exchange, string symbol, DateTime exp) {
+      public Security get_future_security(Venue exchange, string symbol, DateTime exp)
+      {
          mbts.SecurityExpiration sec_exp = new mbts.SecurityExpiration(exchange, SecurityType.Future, symbol, exp);
          Security rv = _fut_map.TryGetValue(sec_exp, out rv) ? rv : null;
          return rv;
       }
 
-      public Security get_spread_security(Venue exchange, string spread_def) {
+      public Security get_spread_security(Venue exchange, string spread_def)
+      {
          IDictionary<string, Security> exch_spread_map;
          Security rv = null;
          if (_spread_map.TryGetValue(exchange, out exch_spread_map))
@@ -88,7 +99,8 @@ namespace BCG {
 
       public IEnumerable<Tuple<Venue, string, uint>> get_spots() { return new List<Tuple<Venue, string, uint>>(); }
 
-      public Security get_o2_security(uint sec_key) {
+      public Security get_o2_security(uint sec_key)
+      {
          Security rv = _key_sec_map.TryGetValue(sec_key, out rv) ? rv : null;
          return rv;
       }
@@ -97,10 +109,12 @@ namespace BCG {
 
       /// Given an ExpStrikeKeyedData it attaches
       /// the call and put security keys
-      public ExpStrikeKeyedData<Triplet<T, uint, uint>> attach_sec_keys<T>(ExpStrikeKeyedData<T> data, string option_name, Venue exchange) {
+      public ExpStrikeKeyedData<Triplet<T, uint, uint>> attach_sec_keys<T>(ExpStrikeKeyedData<T> data, string option_name, Venue exchange)
+      {
          ExpStrikeKeyedData<Triplet<T, uint, uint>> rv = new ExpStrikeKeyedData<Triplet<T, uint, uint>>();
          uint[] uninit_keys = new uint[] { 0, 0 };
-         foreach (Triplet<DateTime, double, T> t in data.get_data_linearly()) {
+         foreach (Triplet<DateTime, double, T> t in data.get_data_linearly())
+         {
             mbts.SecurityStrike k = new mbts.SecurityStrike(new mbts.SecurityExpiration(exchange, SecurityType.Option, option_name, t.first), t.second);
             Security[] cp_keys = _option_map.TryGetValue(k, out cp_keys) ? cp_keys : null;
             uint c_key = cp_keys != null && cp_keys[0] != null ? cp_keys[0].Id : 0,
@@ -114,7 +128,8 @@ namespace BCG {
       //    return new DateTime((int)t.ExpirationYear, (int)t.ExpirationMonth, (int)t.ExpirationDay);
       // }
 
-      public Security fill_partial_security(Security s) {
+      public Security fill_partial_security(Security s)
+      {
          if (s.security_type() == SecurityType.Option && s.Option != null)
             return get_option_security(s.Exchange, s.Symbol, new DateTime((int)s.Option.ExpirationYear, (int)s.Option.ExpirationMonth, (int)s.Option.ExpirationDay), s.Option.Strike, s.Option.CallPut);
          else if (s.security_type() == SecurityType.Future && s.Future != null)
@@ -124,9 +139,11 @@ namespace BCG {
          return null;
       }
 
-      public IEnumerable<Security> get_matching_option_securities(string symbol, Venue exchange) {
+      public IEnumerable<Security> get_matching_option_securities(string symbol, Venue exchange)
+      {
          List<Security> rv = new List<Security>();
-         foreach (var v in _option_map.Where(kv => String.Compare(kv.Key.expiration.symbol, symbol, true) == 0).Select(kv => kv.Value)) {
+         foreach (var v in _option_map.Where(kv => String.Compare(kv.Key.expiration.symbol, symbol, true) == 0).Select(kv => kv.Value))
+         {
             rv.Add(v[0]);
             rv.Add(v[1]);
          }

src/client/spark/NMSheets/Outside/OptionContractKeyedData.cs

@@ -2,41 +2,49 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
-
+using mbts;
 using mbts.spark;
 
-namespace BCG {
-   public class CallPutValue<T> {
+namespace BCG
+{
+   public class CallPutValue<T>
+   {
       public T call_value;
       public T put_value;
-      public void set_value(bool is_call, T t) {
+      public void set_value(bool is_call, T t)
+      {
          if (is_call) call_value = t;
          else put_value = t;
       }
       public T get_value(bool is_call) { return is_call ? call_value : put_value; }
-      public bool value_exists(bool is_call) {
+      public bool value_exists(bool is_call)
+      {
          T v = get_value(is_call);
          return v != null && !v.Equals(default(T));
       }
       public void remove_value(bool is_call) { set_value(is_call, default(T)); }
 
-      public CallPutValue<T> shallow_copy() {
+      public CallPutValue<T> shallow_copy()
+      {
          return new CallPutValue<T>() { call_value = call_value, put_value = put_value };
       }
    }
 
-   public class OptionContractKeyedData<T> {
+   public class OptionContractKeyedData<T>
+   {
       private Dictionary<DateTime, Dictionary<double, CallPutValue<T>>> _map = new Dictionary<DateTime, Dictionary<double, CallPutValue<T>>>();
       public OptionContractKeyedData() { }
 
-      public IReadOnlyDictionary<DateTime, IReadOnlyDictionary<double, CallPutValue<T>>> get_data() {
+      public IReadOnlyDictionary<DateTime, IReadOnlyDictionary<double, CallPutValue<T>>> get_data()
+      {
          Dictionary<DateTime, IReadOnlyDictionary<double, CallPutValue<T>>> rv = new Dictionary<DateTime, IReadOnlyDictionary<double, CallPutValue<T>>>();
          foreach (DateTime exp in _map.Keys)
             rv[exp] = get_exp_data(exp);
          return rv;
       }
 
-      public void setValue(DateTime e, double k, bool is_call, T data) {
+      public void setValue(DateTime e, double k, bool is_call, T data)
+      {
          Dictionary<double, CallPutValue<T>> k_map = null;
          if (!_map.TryGetValue(e, out k_map))
             _map.Add(e, k_map = new Dictionary<double, CallPutValue<T>>());
@@ -46,7 +54,8 @@ namespace BCG {
          k_val.set_value(is_call, data);
       }
 
-      public T getValue(DateTime e, double k, bool is_call) {
+      public T getValue(DateTime e, double k, bool is_call)
+      {
          Dictionary<double, CallPutValue<T>> k_map = null;
          CallPutValue<T> k_val = null;
          if (_map.TryGetValue(e, out k_map))
@@ -55,7 +64,8 @@ namespace BCG {
          return default(T);
       }
 
-      public T get_value(DateTime e, double k, bool is_call, T not_found_val) {
+      public T get_value(DateTime e, double k, bool is_call, T not_found_val)
+      {
          Dictionary<double, CallPutValue<T>> k_map = null;
          CallPutValue<T> k_val = null;
          if (_map.TryGetValue(e, out k_map))
@@ -64,7 +74,8 @@ namespace BCG {
          return not_found_val;
       }
 
-      public void remove(DateTime e, double k, bool is_call) {
+      public void remove(DateTime e, double k, bool is_call)
+      {
          Dictionary<double, CallPutValue<T>> k_map = null;
          CallPutValue<T> k_val = null;
          if (_map.TryGetValue(e, out k_map))
@@ -72,7 +83,8 @@ namespace BCG {
                k_val.remove_value(is_call);
       }
 
-      public bool contains(DateTime e, double k, bool is_call) {
+      public bool contains(DateTime e, double k, bool is_call)
+      {
          Dictionary<double, CallPutValue<T>> k_map = null;
          CallPutValue<T> k_val = null;
          if (_map.TryGetValue(e, out k_map))
@@ -81,14 +93,17 @@ namespace BCG {
          return false;
       }
 
-      public bool contain_exp(DateTime e) {
+      public bool contain_exp(DateTime e)
+      {
          Dictionary<double, CallPutValue<T>> k_map;
          return _map.TryGetValue(e, out k_map) && k_map != null && k_map.Count > 0;
       }
 
-      public List<Triplet<DateTime, double, CallPutValue<T>>> get_linear_data() {
+      public List<Triplet<DateTime, double, CallPutValue<T>>> get_linear_data()
+      {
          List<Triplet<DateTime, double, CallPutValue<T>>> rv = new List<Triplet<DateTime, double, CallPutValue<T>>>();
-         foreach (KeyValuePair<DateTime, Dictionary<double, CallPutValue<T>>> kv1 in _map) {
+         foreach (KeyValuePair<DateTime, Dictionary<double, CallPutValue<T>>> kv1 in _map)
+         {
             DateTime e = kv1.Key;
             foreach (KeyValuePair<double, CallPutValue<T>> kv2 in kv1.Value)
                rv.Add(new Triplet<DateTime, double, CallPutValue<T>>(e, kv2.Key, kv2.Value));
@@ -96,14 +111,17 @@ namespace BCG {
          return rv;
       }
 
-      public IReadOnlyDictionary<double, CallPutValue<T>> get_exp_data(DateTime exp) {
+      public IReadOnlyDictionary<double, CallPutValue<T>> get_exp_data(DateTime exp)
+      {
          Dictionary<double, CallPutValue<T>> exp_map = null;
          return _map.TryGetValue(exp, out exp_map) ? exp_map : new Dictionary<double, CallPutValue<T>>();
       }
 
-      public OptionContractKeyedData<T> shallow_copy() {
+      public OptionContractKeyedData<T> shallow_copy()
+      {
          Dictionary<DateTime, Dictionary<double, CallPutValue<T>>> rv = new Dictionary<DateTime, Dictionary<double, CallPutValue<T>>>();
-         foreach (KeyValuePair<DateTime, Dictionary<double, CallPutValue<T>>> kv1 in _map) {
+         foreach (KeyValuePair<DateTime, Dictionary<double, CallPutValue<T>>> kv1 in _map)
+         {
             Dictionary<double, CallPutValue<T>> k_map = new Dictionary<double, CallPutValue<T>>();
             foreach (KeyValuePair<double, CallPutValue<T>> kv2 in kv1.Value)
                k_map.Add(kv2.Key, kv2.Value.shallow_copy());
@@ -112,15 +130,18 @@ namespace BCG {
          return new OptionContractKeyedData<T>() { _map = rv };
       }
 
-      public void mutate_data(Action<T> mutate_func) {
+      public void mutate_data(Action<T> mutate_func)
+      {
          foreach (Dictionary<double, CallPutValue<T>> k_map in _map.Values)
-            foreach (CallPutValue<T> kp_val in k_map.Values) {
+            foreach (CallPutValue<T> kp_val in k_map.Values)
+            {
                mutate_func(kp_val.call_value);
                mutate_func(kp_val.put_value);
             }
       }
 
-      public IDictionary<DateTime, IEnumerable<double>> get_contracts() {
+      public IDictionary<DateTime, IEnumerable<double>> get_contracts()
+      {
          Dictionary<DateTime, IEnumerable<double>> rv = new Dictionary<DateTime, IEnumerable<double>>();
          foreach (KeyValuePair<DateTime, Dictionary<double, CallPutValue<T>>> k in _map)
             rv.Add(k.Key, k.Value.Keys);

src/client/spark/NMSheets/Outside/tc2/Calculator2/MainForm/ATM/ATMData.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
-
+using mbts;
 using mbts.spark;
 
 namespace BCG {

src/client/spark/NMSheets/Outside/tc2/Calculator2/MainForm/ATM/OptionATMDataSource.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using mbts.ui;
-
+using mbts;
 using BCG.Utilities;
 using mbts.spark;
 

src/client/spark/NMSheets/UIComponents/ExpirationHeaderDriverControl/NMSheetDriverControlStrip.cs

@@ -9,7 +9,7 @@ using BCG.Utilities;
 using mbts.ui.dpi;
 using mbts.spark;
 using bts.api;
-
+using mbts;
 namespace BCG {
    /// Control that sits on top (under the vol control strip).
    /// For every selected option expiration, theres a driver

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetTheoGrid.cs

@@ -7,7 +7,7 @@ using System.Data;
 using System.Windows.Forms;
 using System.Drawing.Imaging;
 using System.Drawing.Drawing2D;
-
+using mbts;
 using BCG.Utilities;
 using BCG.Utilities.General;
 using mbts.spark;

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetTheoGridMouseHandling.cs

@@ -7,7 +7,7 @@ using System.Data;
 using System.Windows.Forms;
 using System.Drawing.Imaging;
 using System.Drawing.Drawing2D;
-
+using mbts;
 using BCG.Utilities;
 using BCG.Utilities.General;
 using mbts.spark;