platformmap needed value for glbx

The build was broken because Spark.csproj was looking for SharedStreams
at some hardcoded path on taha's comp

CMakeLists.txt

@@ -25,7 +25,7 @@ if (WIN32)
   set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
 else()
   set(BUILD_SPARKD ON CACHE BOOL "Build sparkd")
-  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
+  # set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
@@ -117,7 +117,7 @@ else()
   endif()
   add_subdirectory(src/lib)
   if (BUILD_ADMIN_WEB_APP)
-    add_subdirectory(src/Spark.Admin.WebApp)
+    # add_subdirectory(src/Spark.Admin.WebApp)
   endif()
   if(BUILD_DOCS)
     add_subdirectory(doc/user)

CMakePresets.json

@@ -267,7 +267,6 @@
       "hidden": true,
       "targets": [
         "data_files_dir",
-        "admin_web_app_bundle",
         "install",
         "libspark",
         "rapidjson_project",

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionTokens.lex

@@ -14,7 +14,7 @@ ParenthesisClose   \)
 Add                \+
 Subtract           \-
 Multiply           \*
-Divide             \\
+Divide             /
 Equals             =
 Comma              ,
 

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -2,6 +2,8 @@ namespace Bts.BooleanExpressions.Arguments;
 
 using bts;
 
+using Spark.Core.Collections;
+
 static class ArgumentComparer {
    public static int Compare(
       IArgument value1, IArgument value2, IDictionary<string,object?> env) {
@@ -14,7 +16,10 @@ static class ArgumentComparer {
       var cmp1 = value1.Cast<IComparable>();
       var cmp2 = value2.Cast<IComparable>();
       try {
-         return cmp1.GetValue(env).CompareTo(cmp2.GetValue(env));
+         var v1 = cmp1.GetValue(env);
+         var v2 = cmp2.GetValue(env);
+         return v1.GetType().IsNumeric() ?
+            new NumericComparer().Compare(v1, v2) : v1.CompareTo(v2);
       } catch (Exception ex) {
          Log.Ui.error($"Comparison failed: {ex}");
          throw;

src/DataStreamSuite/BooleanExpressions/BooleanExpressionGrammar.y

@@ -44,7 +44,7 @@
 %%
 
 BoundExpression : Definitions Expression { $$ = new($2, $1); }
-   | Expression;
+   | Expression { $$ = new($1, ImmutableList<Definition>.Empty); };
 
 Definitions :
    Definition { $$ = ImmutableList<Definition>.Empty.Add($1); }
@@ -57,46 +57,70 @@ Definition :
 
 Expression :
    Expression OR Expression {
-      $$ = new(env => $1.Evaluate(env) || $3.Evaluate(env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) || right.Evaluate(env));
    }
    | Expression AND Expression {
-      $$ = new(env => $1.Evaluate(env) && $3.Evaluate(env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) && right.Evaluate(env));
    }
    | Expression XOR Expression {
-      $$ = new(env => $1.Evaluate(env) != $3.Evaluate(env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => left.Evaluate(env) != right.Evaluate(env));
    }
    | NOT Expression {
-      $$ = new(env => !$2.Evaluate(env));
+      var exp = $2;
+      $$ = new(env => !exp.Evaluate(env));
    }
    | PARENTHESIS_OPEN Expression PARENTHESIS_CLOSE {
       $$ = $2; 
    }
    | Argument GREATER Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) > 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) > 0);
    }
    | Argument GREATER_EQUAL Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) >= 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) >= 0);
    }
    | Argument EQUAL Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) == 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) == 0);
    }
    | Argument LESS_EQUAL Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) <= 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) <= 0);
    }
    | Argument LESS Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) < 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) < 0);
    }
    | Argument DIFFERENT Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) != 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
    }
    | Argument DIFFERENT2 Argument {
-      $$ = new(env => Arguments.ArgumentComparer.Compare($1, $3, env) != 0);
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentComparer.Compare(left, right, env) != 0);
    }
    | Argument MATCHES Argument {
-      $$ = new(env => Arguments.ArgumentMatcher.Match($1, $3, env));
+      var left = $1;
+      var right = $3;
+      $$ = new(env => Arguments.ArgumentMatcher.Match(left, right, env));
    }
    | Argument {
-      $$ = new(env => (bool)$1.GetValue(env));
+      var arg = $1;
+      $$ = new(env => (bool)arg.GetValue(env));
    }
    ;
 

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -7,7 +7,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
-    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
+    <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />    
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -0,0 +1,32 @@
+using BCG;
+
+namespace BCG
+{
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("add_to_cloud")]
+   public class AddToCloudStream : ViewStream
+   {
+      private readonly string _name, _stream_exp;
+      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
+      public AddToCloudStream(string name, params string[] lines)
+      : this(name, 0, lines)
+      {
+      }
+
+      public AddToCloudStream(string name, int persist, params string[] lines)
+      {
+         _name = name;
+         _stream_exp = getExpression(lines);
+         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
+         mySetRealSource(new TableStream("cloud_streams"));
+      }
+
+      static string getExpression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -0,0 +1,100 @@
+using BCG;
+
+using System;
+using System.Collections.Generic;
+
+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
+using Bts.DataStreamExpressions;
+using System.Collections.Generic;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+   // listens to the requests posted on "cloud_streams" request table, and 
+   // if unfulfilled starts a ViewPubsliher
+   public class CloudStreamManager : IWorker
+   {
+      public class StreamPubUnit
+      {
+         public string name;
+         public string stream_expression;
+         public ViewStream stream;
+         public bool persist;
+         public ViewPublisher view_publisher;
+
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
+         {
+            name = name_;
+            stream_expression = stream_expression_;
+            stream = stream_;
+            persist = persist_;
+            view_publisher = view_publisher_;
+         }
+      }
+      static ViewStream build_stream(string stream_expression)
+      {
+         try
+         {
+            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+            return rv;
+
+         }
+         catch (Exception ex)
+         {
+            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+            return null;
+         }
+      }
+      private IDictionary<string, StreamPubUnit>
+         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
+      private ViewStream _stream = new TableStream("cloud_streams");
+      private PubTableBroker _broker = new PubTableBroker();
+
+      public CloudStreamManager()
+      {
+         _stream.onChange += _on_str;
+
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         foreach (IBlueRow r in dp.newRows)
+         {
+            string name = r.get<string>("name");
+            bool persist = r.get<bool>("persist");
+            string stream_exp = r.get<string>("stream_exp");
+            if (!_stream_pub_map.ContainsKey(name))
+            {
+               _assign(name, stream_exp, persist);
+            }
+         }
+      }
+
+      private void _assign(string name, string stream_expression, bool persist)
+      {
+         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
+         ViewStream stream = build_stream(stream_expression);
+         _expose_to_ws(name, stream);
+         if (persist)
+         {
+            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
+         }
+         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
+         StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
+         _stream_pub_map.Add(name, u);
+      }
+
+      private void _expose_to_ws(string name, ViewStream stream)
+      {
+         // TODO: john
+      }
+
+      public void doWork()
+      {
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -0,0 +1,59 @@
+using BCG;
+
+using System.Linq;
+
+namespace BCG
+{
+   public class StreamOffloadRequestor
+   {
+      public class OffloadOp
+      {
+         string name;
+         string stream_exp;
+      }
+      private IBlueTableSchema _schema;
+      private IBlueRowCollection _rows;
+      private PubTableBroker _broker = new PubTableBroker();
+      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+      public StreamOffloadRequestor()
+      {
+         _schema = new BlueTableSchema("cloud_streams",
+                                   new ColumnInfo("id", 0, typeof(int)),
+                                   new ColumnInfo("name", 1, typeof(string)),
+                                   new ColumnInfo("persist", 2, typeof(bool)),
+                                   new ColumnInfo("stream_exp", 3, typeof(string)));
+         _existing_offloads_vs.onChange += _on_str;
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         _rows = dp.newRows;
+         // this is needed so the broker knows the state of the table
+         _broker.createTable(_rows);
+      }
+      public void add_to_cloud(string name, string stream_exp, bool persist)
+      {
+         // check if it exists
+         if (_rows == null)
+         {
+            BlueTable table = new BlueTable(_schema);
+            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+            table.addRow(nr);
+            _broker.createTable(table);
+            // create table  
+         }
+         else
+         {
+            if (exists(name))
+               return;
+            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+            _broker.addRow(nr);
+         }
+      }
+
+      bool exists(string name)
+      {
+         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/CaptureSnapshotStream.cs

@@ -1,40 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class CaptureSnapshotStream : ViewStream
-//    {
-//       public CaptureSnapshotStream(string basePath,ViewStream sourceStream)
-//       {
-//          myBasePath = basePath;
-//          mySource = sourceStream;
-//          Directory.CreateDirectory(myBasePath);
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DateTime now = DateTime.Now;
-//          new ViewStreamWaiter(new SaverStream(new BlueRowCollectionStaticStream(((DataPacket)newValue).newRows),
-//              Path.Combine(myBasePath, String.Format("{0}_{1}.ds", now.ToString("yyyyMMdd_HHmmss"),
-//              now.Millisecond)))).waitUntilArrived();
-//          myFireEvent(newValue);
-//       }
-
-//       private string myBasePath;
-//       private ViewStream mySource;
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using Google.Protobuf;
+using System.Globalization;
+
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
+   // File naming:
+   // token_epoch_yyyyMMdd_HHmmss
+   // File location: directory(passed)\token\
+   // directory passed on, it creates a folder with the token passed
+   // data:
+   // epochtime, size, serialized data
+   [DataStreamUserRespresentation("pers")]
+   public class PersistorStream : ViewStream {
+      private ViewStream _source_stream;
+      private string _token;
+      private FileStream _out_fs;
+      private IBlueDataSerializer _serializer;
+      private string _file_name = String.Empty;
+      private long _rollover_size;
+      private readonly string _directory_path;
+      private int _n_updates_in_current_file = 0;
+
+      public PersistorStream(ViewStream src, string token, string directory, int rollover_size) {
+         _source_stream = src;
+         _token = token;
+         _directory_path = directory + $@"\{token}\";
+         _open_new_file();
+         _rollover_size = rollover_size;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _source_stream.onChange += _on_change;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _source_stream.onChange -= _on_change;
+         _out_fs.Close();
+      }
+
+
+      public static DateTime parse_datetime_from_filename(string s) {
+         // allbooks_1697054438_2023-10-11_16-00-38
+         string[] toks = s.Split("_");
+         bts.Log.assert(toks.Length == 4);
+         string dt = $"{toks[2]}_{toks[3]}";
+         return parse_datetime(dt);
+      }
+
+      // this is used to parse the string passed the datetime texts passed to cons of the stream 
+      public static DateTime parse_datetime(string s) {
+         // for whatever reason "yyyy-MM-dd_HH-mm-ss" is not parsable using DateTime.ParseExact (it only gets
+         // the Date not the time), "yyyy-MM-dd_HH:mm:ss" is parsable
+         // however a file name can not have : in its name, therefore we need to parse "yyyy-MM-dd_HH-mm-ss"
+         DateTime rv = DateTime.ParseExact(s, PersistorStream.DateTime_format, CultureInfo.InvariantCulture);
+         string[] toks = s.Split("_");
+         string time_str = toks[1];
+         string[] time_toks = time_str.Split("-");
+         bts.Log.assert(time_toks.Length == 3);
+         int hh = Int32.Parse(time_toks[0]);
+         int mm = Int32.Parse(time_toks[1]);
+         int ss = Int32.Parse(time_toks[2]);
+         rv = rv.Date;
+         rv = rv.AddHours(hh);
+         rv = rv.AddMinutes(mm);
+         rv = rv.AddSeconds(ss);
+         return rv;
+      }
+
+      public static string DateTime_format {
+         get {
+            return "yyyy-MM-dd_HH-mm-ss";
+         }
+      }
+
+      public static long dt_to_epoch(DateTime dt) {
+         DateTimeOffset dtos = new DateTimeOffset(dt, TimeSpan.Zero);
+         return dtos.ToUnixTimeSeconds();
+      }
+      public static long epoch_now() {
+         return DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+      }
+
+      void _on_change(object new_value) {
+         DataPacket dp = new_value as DataPacket;
+         byte[] serialized_update = null;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         if (_write_create) {
+            // new files always starts with create
+            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+         } else
+            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
+
+         BinaryWriter bw = new BinaryWriter(_out_fs);
+         long utc_epoch = epoch_now();
+
+         //time, length of data, data
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
+         bw.Write(serialized_update);
+         _out_fs.Flush();
+         myFireEvent(dp);
+         _n_updates_in_current_file++;
+         if (_out_fs.Length > _rollover_size) {
+            _close_current_and_open_new();
+         }
+      }
+
+      void _close_current_and_open_new() {
+         _out_fs.Close();
+         _open_new_file();
+      }
+
+      void _open_new_file() {
+         var epoch = epoch_now();
+         _n_updates_in_current_file = 0;
+         // Check if the directory was successfully created
+         if (!Directory.Exists(_directory_path)) {
+            Directory.CreateDirectory(_directory_path);
+         }
+         _file_name = $"{_directory_path}{_token}_{epoch}_{DateTime.Now.ToString(DateTime_format)}";
+         _out_fs = new FileStream(_file_name, FileMode.OpenOrCreate);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -5,111 +5,184 @@ using System.Text;
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
 using BCG.Utilities.General;
-using BCG.Utilities;
+using bts.collections;
+using System.Reflection.Metadata;
 
-namespace BCG
-{
-   public delegate void PlaybackStreamProgress(long readFar, long totalBytesRead);
-   public class PlaybackStream : ViewStream, IWorker
-   {
 
-      public PlaybackStream(string fileName) : this(fileName, 1.0) { }
+namespace BCG {
 
-      private string myFileName;
-      private FileStream myFileReader;
-      private double myPlaybackSpeed;
-      private DateTime myPlaybackStart;
-      private DateTime myNextPacketTime;
-      private IClientDeserializer myDeserializer;
-      private ITableAccumulator myAccumulator;
+   public class PlaybackStream : ViewStream, IWorker {
+      private readonly PlaybackType _type;
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      // these are the packets already downloaded from the current file
+      private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+      //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+      private IEnumerable<string> _file_names;
+      private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+      private DateTime _playback_t0, _playback_t1;
+      private bool _all_files_read = false;
+      SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
 
-      private BlueTable myLastTableVersion;
-
-      public PlaybackStream(string fileName, double playbackSpeed)
-      {
-         myFileName = fileName;
-         myPlaybackSpeed = playbackSpeed;
+      public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+         _type = PlaybackType.Streaming;
+         _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+         _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+         string token_directory_name = $"{directory_name}\\{token}";
+         bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+         List<string> files_in_range = new List<string>();
+         if (Directory.Exists(token_directory_name)) {
+            _file_names =
+         Directory.GetFiles(token_directory_name).Where(f => {
+            DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+            return file_time > _playback_t0 && file_time < _playback_t1;
+         });
+         }
       }
 
-      protected override void myOnFirstSubscriber()
-      {         myPlaybackStart = DateTime.Now;
-         myNextPacketTime = DateTime.MinValue;
-         myFileReader = new FileStream(myFileName, FileMode.Open, FileAccess.Read, FileShare.Read);
-         myReadNextPacketTime();
-         ViewStreamDatabase.instance.addWorker(this);
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+         _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _snapshot_data.packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
       }
 
-      protected override void myOnLastSubscriber()
-      {
-         if (myFileReader != null)
-            myFileReader.Close();
+
+      private void _load_packets_from_current_file() {
+         int curr_idx = _current_fs.Item1;
+         FileStream curr_fs = _current_fs.Item2;
+         _packets = new List<PlaybackPacket>();
+         _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+      }
+
+      private bool current_file_all_replayed {
+         get {
+            bool rv = true;
+            _packets.ForEach(p => rv = rv && p.replayed);
+            return rv;
+         }
+      }
+
+      protected override void myOnLastSubscriber() {
+         var fs = _current_fs.Item2;
+         if (fs != null) {
+            fs.Close();
+         }
          ViewStreamDatabase.instance.removeWorker(this);
       }
 
-      protected void myReadNextPacketTime()
-      {
-         if (myFileReader.Position == myFileReader.Length)
-         {
-            myNextPacketTime = DateTime.MaxValue;
-            myFileReader.Dispose();
-            if (onFileExhausted != null)
-               onFileExhausted();
-         }
-         else
-         {
-            double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-            myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-            if (onProgress != null)
-               onProgress(myFileReader.Position, myFileReader.Length);
+      // public event PlaybackStreamProgress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
          }
+         return null;
       }
 
-      public event PlaybackStreamProgress onProgress;
-      public event Action onFileExhausted;
-
+      private PlaybackPacket _get_previous_packet(PlaybackPacket p) {
+         if (p.idx == 0)
+            return null;
+         return _packets[p.idx - 1];
+      }
 
       #region IWorker Members
 
-      public void doWork()
-      {
-         if (myNextPacketTime <= DateTime.Now)
-         {
-            if (myDeserializer == null)
-            {
-               myDeserializer = new ClientUniversalDeserializer();
-               myAccumulator = new TableAccumulator(myFileName, null);
-            }
-
-            BinaryReader reader = new BinaryReader(myFileReader);
-            int numBytes = reader.ReadInt32();
-            byte[] thisData = reader.ReadBytes(numBytes);
-            byte[] ops;
-            Array resultRowCollections;
-            myDeserializer.deserialize(new Stream[] { new MemoryStream(thisData) }, out ops, out resultRowCollections);
-
-            //there should only be one op in there becuae we just read a single update.
-            // BlueEventLog.logAssert(ops.Length == 1,"Playback stream got " +ops.Length.ToString() + " op codes in one update, from stream file "+myFileName);
-            //there shoudl only be one set of collections becaue there was just a single op code 
-            //(those two arrays are parallel)
-            // BlueEventLog.logAssert(resultRowCollections.Length == 1, "StreamOpCode was create, serializer should return exactly 1 set of rows");
-
-            if (ops[0] == (byte)StreamOpCode.create)
-            {
-               myLastTableVersion = new BlueTable((resultRowCollections.GetValue(0) as IBlueRowCollection[])[0]);
-               myFireEvent(new DataPacket(myLastTableVersion, new BlueTable(myLastTableVersion.schema), new BlueTable(myLastTableVersion.schema)));
-            }
-            else
-            {
-               // BlueEventLog.logAssert(ops[0] == (byte)StreamOpCode.multipleRemoveAdd, "Playback stream got unexpected open code " + ops[0].ToString() + " from file " + myFileName);
-               // BlueEventLog.logAssert(myLastTableVersion != null, "Playback stream for file "+myFileName + "got a non-create before a create");
-               IBlueRowCollection[] deletedAndAdded = resultRowCollections.GetValue(0) as IBlueRowCollection[];
-               BlueDataUtility.absorbChanges(myLastTableVersion, deletedAndAdded[0], deletedAndAdded[1]);
-               myFireEvent(new DataPacket(myLastTableVersion, deletedAndAdded[1], deletedAndAdded[0]));
-            }
-            
-            myReadNextPacketTime();
+      public void doWork() {
+         if (_all_files_read)
+            return;
+         if (current_file_all_replayed) {
+            _load_next_file();
          }
+         PlaybackPacket curr_packet = _locate_packet_to_replay();
+         if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+            byte[] ops;
+            Array row_array;
+            // what a stupid interface!
+            _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+            _fire_packet(ops, row_array);
+            curr_packet.replay_timestamp = PersistorStream.epoch_now();
+         } else {
+            PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+            // this is the time we need to fire (replay) this packet
+            long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+            long now_epoch = PersistorStream.epoch_now();
+            // fire when we are past the 
+            if (now_epoch >= time_to_fire) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+               _fire_packet(ops, row_array);
+               curr_packet.replay_timestamp = PersistorStream.epoch_now();
+            }
+         }
+      }
 
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
+
+         }
+      }
+
+      private void _load_next_file() {
+         var new_idx = _current_fs.Item1 + 1;
+         if (new_idx >= _file_names.Count()) {
+            _all_files_read = true;
+            bts.Log.Gen.info("All files read");
+            return;
+         }
+         var prev_fs = _current_fs.Item2;
+         if (prev_fs != null)
+            prev_fs.Close();
+         var new_filename = _file_names.ElementAt(new_idx);
+         bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+         var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+         _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+         _load_packets_from_current_file();
       }
 
       #endregion

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -0,0 +1,115 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using bts.collections;
+using System.Reflection.Metadata;
+
+
+namespace BCG {
+   public class SnapshotStream : ViewStream {
+
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      public DateTime _snapshot_time;
+      public string _snapshot_file;
+      public IEnumerable<PlaybackPacket> _packets;
+
+
+      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+         string token_directory_name = $"{directory_name}\\{token}";
+         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+         _packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
+      }
+
+      private void _absorb_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+            } else {
+               bts.Log.assert(false, $"Operator: {op} is not expected");
+            }
+         }
+      }
+
+      protected override void myOnLastSubscriber() {
+      }
+
+      // public event SnapshotStream.Progress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
+         }
+         return null;
+      }
+
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }
+
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -0,0 +1,145 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Globalization;
+using System.IO;
+
+
+namespace BCG {
+   // each stream op in the replay file is reprsented by a packet here
+   public class PlaybackPacket {
+      public readonly int idx;
+      public readonly long timestamp;
+      public readonly byte[] data;
+      public long replay_timestamp = 0; // non zero it has been replayed
+      public long offset_from_previous = 0;
+      public PlaybackPacket(int the_idx, long the_timestamp, byte[] the_data) {
+         this.idx = the_idx;
+         this.timestamp = the_timestamp;
+         this.data = the_data;
+      }
+      public bool replayed {
+         get { return replay_timestamp != 0; }
+      }
+   }
+
+   public enum PlaybackType {
+      Streaming = 0,
+      Snapshot
+   }
+   public class SnapshotData {
+      public DateTime snapshot_time;
+      public string snapshot_file;
+      public SnapshotData(DateTime snapshot_time_arg, string snapshot_file_arg) {
+         snapshot_time = snapshot_time_arg;
+         snapshot_file = snapshot_file_arg;
+      }
+      public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
+   }
+   public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            string next_fn = file_names[i + 1];
+            DateTime f_dt = PersistorStream.parse_datetime_from_filename(fn);
+            DateTime next_f_dt = PersistorStream.parse_datetime_from_filename(next_fn);
+            if (snapshot_time >= f_dt && snapshot_time <= next_f_dt) {
+               return fn;
+            }
+         }
+         return String.Empty;
+      }
+      public static DateTime parse_datetime_from_filename(string s) {
+         // allbooks_1697054438_2023-10-11_16-00-38
+         string[] toks = s.Split("_");
+         bts.Log.assert(toks.Length == 4);
+         string dt = $"{toks[2]}_{toks[3]}";
+         return parse_datetime(dt);
+      }
+
+      // this is used to parse the string passed the datetime texts passed to cons of the stream 
+      public static DateTime parse_datetime(string s) {
+         // for whatever reason "yyyy-MM-dd_HH-mm-ss" is not parsable using DateTime.ParseExact (it only gets
+         // the Date not the time), "yyyy-MM-dd_HH:mm:ss" is parsable
+         // however a file name can not have : in its name, therefore we need to parse "yyyy-MM-dd_HH-mm-ss"
+         DateTime rv = DateTime.ParseExact(s, PersistorStream.DateTime_format, CultureInfo.InvariantCulture);
+         string[] toks = s.Split("_");
+         string time_str = toks[1];
+         string[] time_toks = time_str.Split("-");
+         bts.Log.assert(time_toks.Length == 3);
+         int hh = Int32.Parse(time_toks[0]);
+         int mm = Int32.Parse(time_toks[1]);
+         int ss = Int32.Parse(time_toks[2]);
+         rv = rv.Date;
+         rv = rv.AddHours(hh);
+         rv = rv.AddMinutes(mm);
+         rv = rv.AddSeconds(ss);
+         return rv;
+      }
+
+      public static string DateTime_format {
+         get {
+            return "yyyy-MM-dd_HH-mm-ss";
+         }
+      }
+
+      public static long dt_to_epoch(DateTime dt) {
+         DateTimeOffset dtos = new DateTimeOffset(dt, TimeSpan.Zero);
+         return dtos.ToUnixTimeSeconds();
+      }
+      public static long epoch_now() {
+         return DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+      }
+      public static DateTime epoch_to_dt(long epoch_seconds) {
+         return DateTimeOffset.FromUnixTimeSeconds(epoch_seconds).DateTime;
+      }
+
+      public static PlaybackPacket find_packet_at_time(IEnumerable<PlaybackPacket> packets, DateTime snapshot_time) {
+         log_file_ts(packets, snapshot_time);
+         long snapshot_time_epoch = PersistorStream.dt_to_epoch(snapshot_time);
+         for (int i = 0; i < packets.Count() - 1; i++) {
+            PlaybackPacket curr_p = packets.ElementAt(i);
+            PlaybackPacket next_p = packets.ElementAt(i + 1);
+            if (curr_p.timestamp <= snapshot_time_epoch && next_p.timestamp >= snapshot_time_epoch) {
+               return curr_p;
+            }
+         }
+         // if for some reason it cant find it, send the last packet
+         return packets.Last();
+      }
+
+      private static void log_file_ts(IEnumerable<PlaybackPacket> packets, DateTime snapshot_time) {
+         string big_log_msg = String.Empty;
+         foreach (var p in packets) {
+            big_log_msg += $"packet idx: {p.idx} epoch_ts: {p.timestamp} ts: {epoch_to_dt(p.timestamp)} snapshot_time: {snapshot_time}";
+            big_log_msg += "\n";
+         }
+         bts.Log.Gen.info($"Packets: {big_log_msg}");
+      }
+
+      public static IEnumerable<PlaybackPacket> load_packets_from_file(FileStream fs) {
+         BinaryReader br = new BinaryReader(fs);
+         List<PlaybackPacket> rv = new List<PlaybackPacket>();
+         while (fs.Position < fs.Length) {
+            try {
+               long ts = br.ReadInt64();
+               int sz = br.ReadInt32();
+               byte[] data = br.ReadBytes(sz);
+               var pb = new PlaybackPacket(rv.Count, ts, data);
+               rv.Add(pb);
+            } catch (Exception ex) {
+               bts.Log.Gen.error($"ex: {ex}");
+               break;
+            }
+         }
+         // setting offset field
+         for (int i = 0; i < rv.Count; i++) {
+            if (i == 0)
+               continue;
+            rv[i].offset_from_previous = rv[i].timestamp - rv[i - 1].timestamp;
+         }
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs

@@ -8,6 +8,7 @@ using mbts.spark;
 
 namespace BCG
 {
+   [DataStreamUserRespresentation("fastJoin")]
    public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
    {
       private readonly NonIncJoinSpec[] _join_specs;
@@ -23,7 +24,7 @@ namespace BCG
 
       public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
                               ViewStream right, string right_join_cols, string right_proj)
-         : this(new Triplet<ViewStream, string, string>[] 
+         : this(new Triplet<ViewStream, string, string>[]
                {
                   new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
                   new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
@@ -31,7 +32,7 @@ namespace BCG
       { }
 
       public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
-         : this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
+         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
       { }
 
       public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
@@ -102,8 +103,8 @@ namespace BCG
          {
             List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
             foreach (NonIncJoinBookkeeper d in _book_keeping_data)
-               d.constribute_cols_to_schema (cis);
-            _output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
+               d.constribute_cols_to_schema(cis);
+            _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
          }
       }
 

src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs

@@ -0,0 +1,48 @@
+
+
+using System;
+
+namespace BCG {
+   // Keeps adding changing rows to the stream, hence preserving the history
+   // and creating a time series, for example this can be used to make bbo updates
+   // into a time series of prices/qty 
+   public class DebugStream : ViewStream {
+
+      private ViewStream _stream;
+      private IBlueRowCollection _last_recv_rows = null;
+
+      public DebugStream(ViewStream stream) {
+         _stream = stream;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _stream.onChange += new ChangeDelegate(_onstream_change);
+      }
+      protected override void myOnLastSubscriber() {
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
+      }
+
+      private void _onstream_change(object o) {
+         DataPacket dp = o as DataPacket;
+         BlueDataUtility.validate_table_schema(dp.newRows);
+         BlueDataUtility.validate_table_schema(dp.addedRows);
+         if (_last_recv_rows == null) {
+            _last_recv_rows = dp.newRows;
+         } else {
+            //schema 
+            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.newRows.schema));
+            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.addedRows.schema));
+            bts.Log.assert(BlueDataUtility.compareSchemas(_last_recv_rows.schema, dp.deletedRows.schema));
+            foreach (IBlueRow r in dp.deletedRows) {
+               bts.Log.assert(_last_recv_rows.containsID(r.get<int>("id")));
+            }
+            foreach (IBlueRow r in dp.addedRows) {
+               bts.Log.assert(dp.newRows.containsID(r.get<int>("id")));
+            }
+            _last_recv_rows = dp.newRows;
+         }
+         myFireEvent(o);
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs

@@ -0,0 +1,169 @@
+using Google.Protobuf.WellKnownTypes;
+
+using System;
+using BCG.Utilities.General;
+
+namespace BCG
+{
+   // Keeps adding changing rows to the stream, hence preserving the history
+   // and creating a time series, for example this can be used to make bbo updates
+   // into a time series of prices/qty 
+   public class TimeSeriesStream : ViewStream
+   {
+      private BlueTable _table;
+      private ViewStream _stream;
+      private int _id = 0;
+
+      public TimeSeriesStream(ViewStream stream)
+      {
+         _stream = stream;
+      }
+
+      protected override void myOnFirstSubscriber()
+      {
+         _stream.onChange += new ChangeDelegate(_onstream_change);
+      }
+      protected override void myOnLastSubscriber()
+      {
+         _stream.onChange -= new ChangeDelegate(_onstream_change);
+      }
+
+
+      private void _onstream_change(object o)
+      {
+         DataPacket dp = o as DataPacket;
+         DateTime now = DateTime.Now;
+         if (_table == null)
+         {
+            IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
+            _table = new BlueTable(schema);
+         }
+         BlueTable added_rows = new BlueTable(_table.schema);
+         foreach (IBlueRow r in dp.addedRows)
+         {
+            MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
+            ar.setValue("timestamp", now);
+            ar.setValue("id", _id++);
+            added_rows.addRow(ar);
+         }
+         _table.addRows(added_rows);
+         myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
+
+      }
+   }
+
+   public class TimeBucketStream : ViewStream, IStreamMapper
+   {
+      private ViewStream _src;
+      private readonly string _time_stamp_col;
+      private IBlueTableSchema _schema;
+      private int _bucket_size_seconds;
+
+      public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
+      {
+         _time_stamp_col = time_stamp_col;
+         _bucket_size_seconds = bucket_size_seconds;
+         var vs = new MapStream(input, this);
+         mySetRealSource(vs);
+      }
+
+      #region IStreamMapper Members
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
+      {
+         IBlueTableSchema rv = input_schema;
+         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
+         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
+         return rv;
+
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime ts = row.get<DateTime>(_time_stamp_col);
+         TimeSpan time_of_day = (ts - ts.Date);
+         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
+         double buckets = seconds / _bucket_size_seconds;
+         int bucket_number = Convert.ToInt32(Math.Floor(buckets));
+         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
+         rv.setValue("bucket_number", bucket_number);
+         return rv;
+      }
+
+      #endregion
+   }
+
+   // public class ThrottleStream : ViewStream, IWorker
+   // {
+   //    private ViewStreamDatabase _vsd;
+   //    private ViewStream _stream;
+   //    private int _throttle_ms;
+   //    private TimeSpan _min_fire_period;
+   //    private DateTime _last_Fire_time = DateTime.MinValue;
+   //    private DataPacket _unfired_dp;
+   //    public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
+   //    {
+   //       _vsd = vsd;
+   //       _stream = base_stream;
+   //       _throttle_ms = throttle_ms;
+   //       _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
+   //    }
+
+   //    protected override void myOnFirstSubscriber()
+   //    {
+   //       _vsd.addWorker(this);
+   //       _stream.onChange += new ChangeDelegate(_onstream_change);
+   //    }
+   //    protected override void myOnLastSubscriber()
+   //    {
+   //       _vsd.removeWorker(this);
+   //       _stream.onChange -= new ChangeDelegate(_onstream_change);
+   //    }
+
+   //    private void _onstream_change(object obj)
+   //    {
+   //       DataPacket dp = obj as DataPacket;
+   //       if (DateTime.Now - _last_Fire_time >= _min_fire_period)
+   //       {
+   //          // ok to fire here
+   //       }
+   //       else
+   //       {
+   //          _last_cached_state = dp.newRows;
+   //       }
+   //       _possibly_fire(dp.newRows);
+   //    }
+
+   //    public void doWork()
+   //    {
+   //       if (_unfired_dp != null)
+   //       {
+
+   //       }
+   //    }
+
+   //    private void _possibly_fire()
+   //    {
+   //       if (_unfired_dp != null)
+   //       {
+   //          if (DateTime.Now - _last_fire_time > _min_fire_period)
+   //          {
+   //             DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
+   //             _unfired_dp = null;
+   //             _last_Fire_time = DateTime.Now;
+   //             myFireEvent(dp);
+   //          }
+   //       }
+   //    }
+
+   // }
+
+
+}
+
+
+

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs

@@ -6,7 +6,7 @@ using BCG.Utilities;
 
 namespace BCG
 {
-   [DataStreamUserRespresentation("csvSaver")]
+   [DataStreamUserRespresentation("CSVSaver")]
    public class CSVSaverStream : ViewStream, ISideEffectStream
    {
       private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -0,0 +1,143 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+
+namespace BCG
+{
+   /// <summary>
+   /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
+   /// </summary>
+   public enum DateConversion
+   {
+      ConvertDoubleToDateTime = 0,
+      ChopTime,
+      UTCToCurrent,
+      ConvertStringToDateTime,
+      ConvertDateTimeToMilliseconds,
+      SecondsOfDay
+   }
+
+   [DataStreamUserRespresentation("dateStandardize")]
+   public class DateStandardizationStream : ViewStream, IStreamMapper
+   {
+      private IBlueTableSchema mySchema;
+      private DateConversion myConversion;
+      private string myConversionColName;
+      private readonly Dictionary<DateConversion, Type> _date_conversion_to_type_map = new Dictionary<DateConversion, Type>()
+      {
+         {DateConversion.ConvertDoubleToDateTime      , typeof(DateTime)},
+         {DateConversion.ChopTime                     , typeof(DateTime)},
+         {DateConversion.UTCToCurrent                 , typeof(DateTime)},
+         {DateConversion.ConvertStringToDateTime      , typeof(DateTime)},
+         {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
+         {DateConversion.SecondsOfDay                 , typeof(Double)}
+      };
+
+
+      private static DateConversion _parse(string s)
+      {
+         DateConversion rv = DateConversion.ChopTime;
+         Enum.TryParse<DateConversion>(s, false, out rv);
+         return rv;
+      }
+      /// <summary>
+      /// for DSQA
+      /// </summary>
+      public DateStandardizationStream(ViewStream vs, string colName, string conversion)
+         : this(vs, colName, _parse(conversion))
+      { }
+
+      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
+      {
+         myConversionColName = colName;
+         myConversion = conversion;
+         mySetRealSource(new MapStream(vs, this));
+      }
+
+      #region IStreamMapper Members
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
+      {
+         if (mySchema == null)
+         {
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (ColumnInfo c in inputSchema.columnInfo)
+            {
+               if (c.colName == myConversionColName)
+               {
+                  Type convType;
+                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
+                  {
+                     //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
+                     convType = typeof(DateTime);
+                  }
+                  cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
+               }
+               else
+                  cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
+            }
+            mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
+         }
+         return mySchema;
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
+      }
+
+      #endregion
+
+      public static object mapDateTime(object objIn, DateConversion conv)
+      {
+         if (objIn == DBNull.Value || objIn == null)
+         {
+            return DBNull.Value;
+         }
+         else
+         {
+            if (conv == DateConversion.ChopTime)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               return dtIn.Date;
+            }
+            else if (conv == DateConversion.ConvertDoubleToDateTime)
+            {
+               double dtIn = (double)objIn;
+               return DateTime.FromOADate(dtIn);
+            }
+            else if (conv == DateConversion.UTCToCurrent)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               return dtIn.ToLocalTime();
+            }
+            else if (conv == DateConversion.ConvertStringToDateTime)
+            {
+               return DateTime.Parse((string)objIn);
+            }
+            else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               return dtIn.TimeOfDay.TotalMilliseconds;
+            }
+            else if (conv == DateConversion.SecondsOfDay)
+            {
+               DateTime dtIn = (DateTime)objIn;
+               DateTime start_of_day = dtIn.Date;
+               TimeSpan day_period = dtIn - start_of_day;
+               return day_period.TotalSeconds;
+            }
+            else
+            {
+               bts.Log.Gen.info("incorrect type");
+               return null;
+            }
+         }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Collections;
+
+namespace BCG
+{
+   [DataStreamUserRepresentation("timeBucket")]
+   public class TimeBucketStream : ViewStream, IStreamMapper
+   {
+      private ViewStream _src;
+      private readonly string _time_stamp_col;
+      private IBlueTableSchema _schema;
+      private uint _bucket_size_seconds;
+
+      public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
+      {
+         _time_stamp_col = time_stamp_col;
+         _bucket_size_seconds = bucket_size_seconds;
+      }
+
+      #region IStreamMapper Members
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
+      {
+         IBlueTableSchema rv = input_schema;
+         //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
+         rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
+         return rv;
+
+      }
+
+      #endregion
+
+      #region IDynamicStreamMapper Members
+
+      public IBlueRow map(IBlueRow row)
+      {
+         DateTime ts = row.get<DateTime>(_time_stamp_col);
+         TimeSpan time_of_day = (ts - ts.Date)
+         int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
+         int buckets = seconds / _bucket_size_seconds;
+         int bucket_number = Math.Floor(buckets);
+         MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
+         rv.setValue("bucket_number", bucket_number);
+      }
+
+      #endregion
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs

@@ -1,10 +1,9 @@
 using System;
+
 using BCG.Utilities.General;
 
-namespace BCG
-{
-	public abstract class TimeUnitStream : ViewStream, IWorker
-	{
+namespace BCG {
+   public abstract class TimeUnitStream : ViewStream, IWorker {
       protected TimeSpan myOffset;
       protected DateTime myConstructionTime;
       protected int mySizeOfStream;
@@ -13,87 +12,80 @@ namespace BCG
       protected ViewStreamDatabase myDb;
       protected int myId = 0;
 
-      public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
-		{
+      public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
          myConstructionTime = DateTime.Now;
          myZeroTime = zeroTime;
          myOffset = offSet;
          mySizeOfStream = sizeOfStream;
          myDb = db;
-         myCollection = new CircularRowCollection (
-            new BlueTableSchema ("Time",
-            new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))), 
+         myCollection = new CircularRowCollection(
+            new BlueTableSchema("Time",
+            new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
             mySizeOfStream);
-		}
+      }
 
       #region ViewStream overriden methods
 
-      protected override void myOnFirstSubscriber()
-      {
-         myDb.addWorker (this);
-         myInit ();
+      protected override void myOnFirstSubscriber() {
+         myDb.addWorker(this);
+         myInit();
       }
 
-      protected override void myOnLastSubscriber()
-      {
-         myDb.removeWorker (this);
+      protected override void myOnLastSubscriber() {
+         myDb.removeWorker(this);
       }
 
       #endregion
 
-      protected virtual void myInit ()
-      {
+      protected virtual void myInit() {
          int periods = (int)
-            (DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
-            
+            (DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
 
-         DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) : 
-            myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
+
+         DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
+            myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
 
 
          if (myGetLatestRow() == null ||
-            !myGetLatestRow()["time"].Equals(dt))
-         {
-            IBlueRow ar = new BlueRow (myCollection.schema,
+            !myGetLatestRow()["time"].Equals(dt)) {
+            IBlueRow ar = new BlueRow(myCollection.schema,
                myId++, dt);
-            IBlueRow dr = myCollection.addRowWithReturn (ar);
-            myFireEvent (new DataPacket (myCollection,
-               myConvertToCollection (ar),
-               myConvertToCollection (dr)));
+            IBlueRow dr = myCollection.addRowWithReturn(ar);
+            if (n_updates <= 3000)
+               myFireEvent(new DataPacket(myCollection,
+                  myConvertToCollection(ar),
+                  myConvertToCollection(dr)));
          }
       }
 
       #region IWorker methods
 
-      public void doWork ()
-      {
-         myInit ();
+      public void doWork() {
+         myInit();
       }
 
       #endregion
 
-      protected abstract DateTime myRoundTime (DateTime dt);
-         
+      protected abstract DateTime myRoundTime(DateTime dt);
+
 
       #region Private Helper methods
 
-      private IBlueRow myGetLatestRow ()
-      {
+      private IBlueRow myGetLatestRow() {
          return myCollection.Count == 0 ?
             null :
-            BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
+            BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
       }
 
-      private IBlueRowCollection myConvertToCollection (IBlueRow r)
-      {
-         BlueTable retVal = new BlueTable (myCollection.schema);
+      private IBlueRowCollection myConvertToCollection(IBlueRow r) {
+         BlueTable retVal = new BlueTable(myCollection.schema);
          if (r != null)
-            retVal.addRow (r);
+            retVal.addRow(r);
          return retVal;
       }
 
       #endregion
 
-	}
+   }
 
 }

src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs

@@ -1,72 +1,75 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Xml.Serialization;
 
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
-   /// This class will be used by the clients to publish data to the pubsub
+   /// This class will be used by the clients to publish tabular data to the pubsub
    /// </summary>
-   public class PubTableBroker 
-   {
-      protected PublisherCore myPublisherCore;
-      protected IDictionary myPublisherMap = new Hashtable ();
-      protected IDictionary mySchemaMap = new Hashtable ();
-      
+   public class PubTableBroker {
+      protected PublisherCore _publisher_core;
+      protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
+      protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
+
       protected bool myWaitingForFirstOp = false;
 
-      public PubTableBroker ()  
-         : this (new PublisherCore()) 
-      {}
+      public PubTableBroker()
+         : this(new PublisherCore()) { }
 
-      public PubTableBroker (PublisherCore publisher)   
-      {myPublisherCore = publisher;}
+      public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
 
-      public PubTableBroker (IBlueRowCollection[] tablesToPublish) 
-         : this (new PublisherCore(), tablesToPublish)
-      {}
+      public PubTableBroker(IBlueRowCollection[] tablesToPublish)
+         : this(new PublisherCore(), tablesToPublish) { }
 
-      public PubTableBroker (PublisherCore publisher, IBlueRowCollection[] tablesToPublish)
-      {
-         myPublisherCore = publisher;
+      public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
+         _publisher_core = publisher;
 
-         myPublisherCore.beginTransaction ();
+         _publisher_core.beginTransaction();
          foreach (IBlueRowCollection table in tablesToPublish)
-            createTable (table);
-         myPublisherCore.commitTransaction ();
+            createTable(table);
+         _publisher_core.commitTransaction();
       }
 
-      public void beginTransaction ()
-      {
+      public void beginTransaction() {
          myWaitingForFirstOp = true;
       }
 
-      public void commitTransaction ()
-      {
+      public void commitTransaction() {
+         bts.Log.Gen.info("PubTableBroker.commitTransaction");
          if (myWaitingForFirstOp)
             myWaitingForFirstOp = false;
          else
-            myPublisherCore.commitTransaction ();
+            _publisher_core.commitTransaction();
       }
 
-      public void rollbackTransaction ()
-      {
+      public void rollbackTransaction() {
+         bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
          if (myWaitingForFirstOp)
             myWaitingForFirstOp = false;
          else
-            myPublisherCore.abortTransaction ();
+            _publisher_core.abortTransaction();
       }
 
-      public virtual void createTable (IBlueRowCollection newTable)
-      {
-         myCheckTransaction();
-         if (myPublisherMap [newTable.schema.tableName.ToLower()] == null)
-         {
-            myPublisherMap [newTable.schema.tableName.ToLower()] = myGetIPubTablePublisher ();
-            mySchemaMap [newTable.schema.tableName.ToLower()] = newTable.schema;
+      public virtual void createTable(IBlueRowCollection newTable) {
+         string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
+         foreach (IBlueRow r in newTable) {
+            log_msg += $"id: {r["id"]}";
          }
-         (myPublisherMap[newTable.schema.tableName.ToLower()] as IPubTablePublisher).createTable (newTable);
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(newTable.schema);
+         myCheckTransaction();
+         IPubTablePublisher table_publisher = null;
+         var name_key = newTable.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
+            table_publisher = _create_pubtable_publisher(name_key);
+            _publisher_map.Add(name_key, table_publisher);
+            _schema_map.Add(name_key, newTable.schema);
+         }
+         table_publisher.createTable(newTable);
       }
 
       /// <summary>
@@ -74,13 +77,17 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void addRow (IBlueRow r)
-      {
+      public virtual void addRow(IBlueRow r) {
+         string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(r.schema);
          myCheckTransaction();
-         if (myPublisherMap[r.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Row can not be added, as the table that has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = r.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Row can not be added, as the table that has not been created yet");
          else
-            (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).addRow (r);
+            table_publisher.addRow(r);
       }
 
       /// <summary>
@@ -88,13 +95,17 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void deleteRow (IBlueRow r)
-      {
+      public virtual void deleteRow(IBlueRow r) {
+         string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(r.schema);
          myCheckTransaction();
-         if (myPublisherMap[r.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Row can not be removed, as the table has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = r.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Row can not be removed, as the table has not been created yet");
          else
-            (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).deleteRow (r);
+            table_publisher.deleteRow(r);
       }
 
       /// <summary>
@@ -102,13 +113,17 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void changeRow (IBlueRow r)
-      {
+      public virtual void changeRow(IBlueRow r) {
+         string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(r.schema);
          myCheckTransaction();
-         if (myPublisherMap[r.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Row can not be changed, as the table has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = r.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Row can not be changed, as the table has not been created yet");
          else
-            (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).changeRow (r);
+            table_publisher.changeRow(r);
       }
 
       /// <summary>
@@ -116,40 +131,54 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void makeBatchChanges (IBlueRowCollection newRows, 
-                                             IBlueRowCollection deletedRows, 
-                                             IBlueRowCollection addedRows)
-      {
-         myCheckTransaction();
-         if (myPublisherMap[newRows.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Batch changes can not be made, as the table has not been created yet");
+      public virtual void makeBatchChanges(IBlueRowCollection newRows,
+                                             IBlueRowCollection deletedRows,
+                                             IBlueRowCollection addedRows) {
+         string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
+         foreach (IBlueRow r in deletedRows)
+            log_msg += $"id: {r["id"]} ";
+         log_msg += "addedRows: ";
+         foreach (IBlueRow r in addedRows)
+            log_msg += $"id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         _check_schema_consistency(newRows.schema);
+         _check_schema_consistency(deletedRows.schema);
+         _check_schema_consistency(addedRows.schema);
+         IPubTablePublisher table_publisher = null;
+         var name_key = newRows.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
          else
-            (myPublisherMap [newRows.schema.tableName.ToLower()] as IPubTablePublisher).batchRemoveAdd (deletedRows, addedRows);
+            table_publisher.batchRemoveAdd(deletedRows, addedRows);
       }
 
-      public virtual void deleteTableRows (string tableName)
-      {
+      public virtual void deleteTableRows(string tableName) {
          myCheckTransaction();
-         (myPublisherMap [tableName.ToLower()] as IPubTablePublisher).createTable (
-            new BlueTable (mySchemaMap[tableName.ToLower()] as IBlueTableSchema));
+         (_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
+            new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
       }
 
-      public int getNumberOfWaitingMessages() { return myPublisherCore.getNumberOfWaitingMessages(); }
-   
-      public void clearTable(string tableName)
-      {myPublisherCore.clearState(PubStreamPrefixer.prefix(tableName));}
+      public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
 
-      protected void myCheckTransaction()
-      {
+      public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
+
+      protected void myCheckTransaction() {
          if (myWaitingForFirstOp)
-            myPublisherCore.beginTransaction ();
+            _publisher_core.beginTransaction();
          myWaitingForFirstOp = false;
       }
 
-      protected virtual IPubTablePublisher myGetIPubTablePublisher ()
-      {
-         return new PubTablePublisher (myPublisherCore);
+      protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
+         bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
+         return new PubTablePublisher(tag, _publisher_core);
+      }
+
+      private void _check_schema_consistency(IBlueTableSchema schema_arg) {
+         IBlueTableSchema schema;
+         if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
+            bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
+         }
       }
 
    }
-   }
+}

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -4,62 +4,51 @@ using System.Data;
 
 using BCG.Utilities;
 
-namespace BCG
-{
-	public class ViewPublisher : IDisposable
-	{
-      protected ViewPublisher () {}
+namespace BCG {
+   public class ViewPublisher : IDisposable {
+      protected ViewPublisher() { }
 
       /// If an name IsNullOrEmpty then it uses the name
       /// in the schema (be careful with that because this
       /// could lead to double publishes if the schema name
       /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker br, ViewStream vs)
-      {
+      public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
          myName = name;
-         myView = new TableStream(name); // TODOds String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
+         myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
          myBroker = br;
          myFirstTime = true;
          myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      public void Dispose()
-      {
+      public void Dispose() {
          myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
          if (!String.IsNullOrEmpty(myName))
             myBroker.clearTable(myName);
       }
 
-      private void myChangeHandler(object o)
-      {
-         DataPacket dp = (DataPacket) o;
-         if (myFirstTime) 
-         {
+      private void myChangeHandler(object o) {
+         DataPacket dp = (DataPacket)o;
+         if (myFirstTime) {
             myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
             myLastRowCount = dp.newRows.Count;
             myBroker.createTable(dp.newRows);
             myFirstTime = false;
-         }
-         else 
-         {
-            if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) 
-            {
-               myLastRowCount  = dp.newRows.Count;
-               myBroker.makeBatchChanges(dp.newRows,dp.deletedRows, dp.addedRows);
+         } else {
+            if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
+               myLastRowCount = dp.newRows.Count;
+               myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
             }
          }
       }
 
-      public int getLastRowPublishedCount()
-      {return myLastRowCount;}
+      public int getLastRowPublishedCount() { return myLastRowCount; }
 
-      public bool hasPublished()
-      { return !myFirstTime; }
+      public bool hasPublished() { return !myFirstTime; }
 
       private int myLastRowCount = -1;
       private string myName;
       private PubTableBroker myBroker;
       protected ViewStream myView;
       private bool myFirstTime;
-	}
+   }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -12,6 +12,7 @@ using BCG.Utilities.General;
 using System.Linq.Expressions;
 //using mbts.spark;
 using mbts;
+using BCG.Extensions;
 
 namespace BCG
 {
@@ -42,6 +43,20 @@ namespace BCG
       {
          return get_row_value(r, col_no, err_value);
       }
+      public static IEnumerable<T> get_covalues<T>(this IBlueRowCollection rows, string col)
+      {
+         List<T> rv = new List<T>();
+         foreach (IBlueRow r in rows)
+            rv.Add(r.get<T>(col));
+         return rv;
+      }
+      public static IEnumerable<T> get_covalues<T>(IEnumerable<IBlueRow> rows, string col)
+      {
+         List<T> rv = new List<T>();
+         foreach (IBlueRow r in rows)
+            rv.Add(r.get<T>(col));
+         return rv;
+      }
 
       public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func)
       {
@@ -85,6 +100,30 @@ namespace BCG
          }
       }
 
+      public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp)
+      {
+         IBlueRowCollection added_rows = new_rows;
+         IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows;
+         return new DataPacket(new_rows, added_rows, deleted_rows);
+      }
+
+      public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp)
+      {
+         if (last_fired_dp == null)
+         {
+            IBlueRowCollection added_rows = new_rows;
+            IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema);
+            return new DataPacket(new_rows, added_rows, deleted_rows);
+         }
+         else
+         {
+            IBlueRowCollection added_rows = null;
+            IBlueRowCollection deleted_rows = null;
+            compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows);
+            return new DataPacket(new_rows, added_rows, deleted_rows);
+         }
+      }
+
       public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows,
                                                              bool handle_datetime_min = false,
                                                              bool handle_double_nan = false)
@@ -112,6 +151,41 @@ namespace BCG
 
       #endregion
 
+      public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width)
+      {
+         var rv = get_printed_rows(table, get_col_width);
+         Console.WriteLine(rv);
+      }
+
+      public static string get_printed_rows(this IBlueRowCollection table, Func<string, int> get_col_width)
+      {
+         // Print column headers
+         string rv = String.Empty;// = $"{table.schema.tableName}______________________________________";
+         foreach (var col_name in table.schema.columnNames)
+         {
+            int col_width = get_col_width(col_name);
+            rv += $"{col_name.PadRight(col_width)}  ";
+         }
+         rv += "\n";
+         //rv += "______________________________________";
+
+         // Print table data
+         foreach (IBlueRow row in table)
+         {
+            foreach (string col_name in table.schema.columnNames)
+            {
+               int col_width = get_col_width(col_name);
+               object row_val = row[col_name];
+               string row_val_str = row_val.ToString();
+               if (row_val != null)
+               {
+                  rv += $"{row_val_str.PadRight(col_width)} ";
+               }
+            }
+            rv += "\n";
+         }
+         return rv;
+      }
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
       {
          { TypeCode.Boolean, typeof(bool) },
@@ -891,16 +965,23 @@ namespace BCG
          compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
       }
 
+      public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
+                                                      out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows)
+      {
+         int[] col_indices = new int[] { new_rows.schema["id"].colNo };
+         compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
+      }
+
       public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices,
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows)
       {
          Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) =>
-            {
-               foreach (int col_no in col_indices)
-                  if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
-                     return false;
-               return true;
-            };
+         {
+            foreach (int col_no in col_indices)
+               if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
+                  return false;
+            return true;
+         };
 
          BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema);
          // adds + changes
@@ -1266,7 +1347,7 @@ namespace BCG
                foreach (ColumnInfo ci1 in s1.columnInfo)
                {
                   ColumnInfo ci2 = s2[ci1.colNo];
-                  if (ci1.colName != ci2.colName || ci1.colType != ci2.colType)
+                  if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo)
                   {
                      result = false;
                      break;
@@ -1280,6 +1361,25 @@ namespace BCG
          return result;
       }
 
+      public static void validate_row_schema(IBlueRow r)
+      {
+         foreach (var ci in r.schema.columnInfo)
+         {
+            object rval = r[ci.colName];
+            if (rval == null)
+               continue;
+            bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType));
+         }
+      }
+
+      public static void validate_table_schema(IBlueRowCollection rows)
+      {
+         foreach (IBlueRow r in rows)
+         {
+            validate_row_schema(r);
+         }
+      }
+
       public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals)
       {
          IBlueRowCollection dels = findMatchingRows(table, cols, vals);

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -5,14 +5,12 @@ using System.Data;
 //using BCG.Utilities;
 using System.Linq;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Anologue of the dotNet DataTable.
    /// </summary>
    [Serializable]
-   public class BlueTable : IBlueRowCollection
-   {
+   public class BlueTable : IBlueRowCollection {
       /// <summary>
       /// Hashtable indexed on "id" to speed up removes
       /// </summary>
@@ -29,22 +27,19 @@ namespace BCG
       /// all the data over
       /// </summary>
       /// <param name="table"></param>
-      public BlueTable(DataTable table)
-      {
+      public BlueTable(DataTable table) {
          ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
          int i = 0;
          foreach (DataColumn col in table.Columns)
             cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType);
          mySchema = new BlueTableSchema(cis, table.TableName);
-         foreach (DataRow row in table.Rows)
-         {
+         foreach (DataRow row in table.Rows) {
             IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
             myIdIndex[newRow["id"]] = newRow;
          }
       }
 
-      public BlueTable(IBlueRowCollection table)
-      {
+      public BlueTable(IBlueRowCollection table) {
          mySchema = table.schema;
          foreach (IBlueRow row in table)
             addRow(row);
@@ -52,12 +47,10 @@ namespace BCG
 
       public BlueTable(string tableName,
           IEnumerable objects,
-          params CMap[] columnMappers)
-      {
+          params CMap[] columnMappers) {
          // extract schema from mappers
          ColumnInfo[] cols = new ColumnInfo[columnMappers.Length];
-         for (int i = 0; i < columnMappers.Length; i++)
-         {
+         for (int i = 0; i < columnMappers.Length; i++) {
             cols[i] = new ColumnInfo(
                 columnMappers[i].getColumnName(), i,
                 columnMappers[i].getColumnType());
@@ -66,8 +59,7 @@ namespace BCG
 
          // load rows
          int rowNumber = 0;
-         foreach (object rowObject in objects)
-         {
+         foreach (object rowObject in objects) {
             object[] values = new object[columnMappers.Length];
             for (int cn = 0; cn < columnMappers.Length; cn++)
                values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject);
@@ -87,10 +79,8 @@ namespace BCG
       /// <summary>
       /// Returns the row by the index, interface property
       /// </summary>
-      public virtual IBlueRow this[int id]
-      {
-         get
-         {
+      public virtual IBlueRow this[int id] {
+         get {
             object row = myIdIndex[id];
             if (row == null)
                throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
@@ -105,8 +95,7 @@ namespace BCG
 
       public virtual bool containsID(int id) { return myIdIndex[id] != null; }
 
-      public virtual bool tryGetValue(int id, out IBlueRow row)
-      {
+      public virtual bool tryGetValue(int id, out IBlueRow row) {
          row = (IBlueRow)myIdIndex[id];
          return (row != null);
       }
@@ -114,8 +103,7 @@ namespace BCG
       #endregion
 
       #region BlueTable members
-      public virtual void clear()
-      {
+      public virtual void clear() {
          myIdIndex.Clear();
       }
 
@@ -123,8 +111,7 @@ namespace BCG
       /// If a row with the same id exists, it replaces it
       /// </summary>
       /// <param name="r"></param>
-      public virtual void forceAddRow(IBlueRow r)
-      {
+      public virtual void forceAddRow(IBlueRow r) {
          myIdIndex[r["id"]] = r;
       }
 
@@ -132,44 +119,41 @@ namespace BCG
       /// Adds a new row to the table. 
       /// </summary>
       /// <param name="row"></param>
-      public virtual void addRow(IBlueRow row)
-      {
-         object val = row["id"];
+      public virtual void addRow(IBlueRow row) {
+         object id = row["id"];
 
-         if (val == null || val == DBNull.Value)
+         if (id == null || id == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
-         try
-         {
-            myIdIndex.Add(val, row);
-         }
-         catch (ArgumentException)
-         {
-            throw new BlueDataException("ID already exists in the table " + mySchema.tableName +
-               " , duplication of field ID not allowed");
+         try {
+            myIdIndex.Add(id, row);
+         } catch (ArgumentException) {
+            throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}");
          }
 
       }
 
-      public virtual void addRow(DataRow row)
-      {
+      public virtual void addRow(DataRow row) {
          object val = row["id"];
 
          if (val == null || val == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
-         if (myIdIndex[val] != null)
-            throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed");
+         if (myIdIndex[val] != null) {
+            string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed";
+            // temp TODOds remove it
+            bts.Log.assert(false, err_msg);
+            throw new BlueDataException(err_msg);
+
+         }
          IBlueRow newRow = new BlueRow(mySchema, row);
          myIdIndex[val] = newRow;
       }
 
-      public virtual void addRows(IBlueRowCollection rows)
-      {
+      public virtual void addRows(IBlueRowCollection rows) {
          foreach (IBlueRow nr in rows)
             addRow(nr);
       }
 
-      public virtual void addRows(IEnumerable rows)
-      {
+      public virtual void addRows(IEnumerable rows) {
          foreach (IBlueRow nr in rows)
             addRow(nr);
       }
@@ -186,37 +170,32 @@ namespace BCG
       /// Removes a row specified by the id. The index helps in making this operation O(1)
       /// </summary>
       /// <param name="id"></param>
-      public virtual bool deleteRow(int id)
-      {
+      public virtual bool deleteRow(int id) {
          if (myIdIndex[id] == null)
             return false;
          myIdIndex.Remove(id);
          return true;
       }
 
-      public virtual bool deleteRows(IBlueRowCollection rows)
-      {
+      public virtual bool deleteRows(IBlueRowCollection rows) {
          bool retVal = true;
          foreach (IBlueRow r in rows)
             retVal &= deleteRow(r);
          return retVal;
       }
 
-      public virtual bool deleteRows(IEnumerable rows)
-      {
+      public virtual bool deleteRows(IEnumerable rows) {
          bool retVal = true;
          foreach (IBlueRow r in rows)
             retVal &= deleteRow(r);
          return retVal;
       }
 
-      public virtual void changeRow(IBlueRow r)
-      {
+      public virtual void changeRow(IBlueRow r) {
          myIdIndex[r["id"]] = r;
       }
 
-      public virtual void changeRows(IBlueRowCollection rows)
-      {
+      public virtual void changeRows(IBlueRowCollection rows) {
          foreach (IBlueRow r in rows)
             changeRow(r);
       }

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -1,5 +1,4 @@
-namespace BCG
-{
+namespace BCG {
    using Bts.DataStream;
    using bts;
 
@@ -10,28 +9,26 @@ namespace BCG
    using System.Reflection;
    using System.IO;
 
-   public class PrimitiveManager
-   {
+   public class PrimitiveManager {
       private IDictionary myTypeLookupTable = new Hashtable();
       private static PrimitiveManager mySelf;
       private ArrayList myAssemblyNames = new ArrayList();
 
-      private PrimitiveManager()
-      {
+      private PrimitiveManager() {
          myLoadAllTypes();
       }
 
-      private void myLoadAllTypes()
-      {
+      private void myLoadAllTypes() {
          AppDomain currentDomain = AppDomain.CurrentDomain;
          currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
          loadAssembly(this.GetType().Assembly.FullName);
-
-         foreach (string assemblyName in Configuration.get<DataStreamConfig>().Assemblies)
-         {
+         var assemblies = new string[] { "datastream.dll", "sharedstreams.dll" };
+         var config_assemblies = Configuration.get<DataStreamConfig>().Assemblies;
+         bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}");
+         foreach (string assemblyName in assemblies/*config_assemblies*/) {
+            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
             string resolvedAssemblyPath = assemblyName;
-            if (!System.IO.Path.IsPathRooted(assemblyName))
-            {
+            if (!System.IO.Path.IsPathRooted(assemblyName)) {
                string appPath = Path.GetDirectoryName(
                               Assembly.GetAssembly(typeof(PrimitiveManager)).CodeBase.Substring("file:///".Length));
                resolvedAssemblyPath = System.IO.Path.GetFullPath(System.IO.Path.Combine(appPath, assemblyName));
@@ -40,8 +37,7 @@ namespace BCG
          }
       }
 
-      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
-      {
+      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
          foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
             if (args.Name.Equals(a.FullName))
                return a;
@@ -52,8 +48,8 @@ namespace BCG
       /// Loads all the Stream types associated with the assembly
       /// </summary>
       /// <param name="assemblyName"></param>
-      public void loadAssembly(string assemblyName)
-      {
+      public void loadAssembly(string assemblyName) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
          if (myAssemblyNames.Contains(assemblyName))
             return;
          myAssemblyNames.Add(assemblyName);
@@ -65,13 +61,11 @@ namespace BCG
             if (!type.IsInterface
                && !type.IsAbstract
                && type.IsSubclassOf(typeof(ViewStream))
-               && type.IsPublic)
-            {
+               && type.IsPublic) {
                bool isVisible = true;
                string userRep = createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
-                  if (att is DataStreamUserRespresentation)
-                  {
+                  if (att is DataStreamUserRespresentation) {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
                      isVisible = dsatt.isVisible;
                      userRep = isVisible ?
@@ -79,16 +73,14 @@ namespace BCG
                         : null;
                      break;
                   }
-               if (isVisible)
-               {
+               if (isVisible) {
                   Log.assert(myTypeLookupTable[userRep] == null);
                   myTypeLookupTable[userRep] = type;
                }
             }
       }
 
-      public static string createDefaultName(string typeName)
-      {
+      public static string createDefaultName(string typeName) {
          if (!typeName.EndsWith("Stream"))
             return typeName;
          string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
@@ -100,8 +92,7 @@ namespace BCG
 
 
 
-      public static PrimitiveManager instance
-      {
+      public static PrimitiveManager instance {
          get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
       }
 
@@ -110,8 +101,7 @@ namespace BCG
       /// </summary>
       /// <param name="friendlyName"></param>
       /// <returns></returns>
-      public string find(string friendlyName)
-      {
+      public string find(string friendlyName) {
          if (myTypeLookupTable[friendlyName] == null)
             throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          return (myTypeLookupTable[friendlyName] as Type).Name;
@@ -124,11 +114,9 @@ namespace BCG
       /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
       /// is the desired one</param>
       /// <returns></returns>
-      public ViewStream build(string friendlyName, ICollection args)
-      {
+      public ViewStream build(string friendlyName, ICollection args) {
          object[] constructorArgs = null;
-         if (args != null && args.Count > 0)
-         {
+         if (args != null && args.Count > 0) {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
@@ -144,8 +132,7 @@ namespace BCG
       /// Returns the name of all the primitives 
       /// </summary>
       /// <returns></returns>
-      public string[] getPrimitiveNames()
-      {
+      public string[] getPrimitiveNames() {
          string[] retVal = new string[myTypeLookupTable.Count];
          myTypeLookupTable.Keys.CopyTo(retVal, 0);
          return retVal;
@@ -155,20 +142,17 @@ namespace BCG
       /// Return the types of all the streams
       /// </summary>
       /// <returns></returns>
-      public Type[] getStreamTypes()
-      {
+      public Type[] getStreamTypes() {
          Type[] retVal = new Type[myTypeLookupTable.Count];
          myTypeLookupTable.Values.CopyTo(retVal, 0);
          return retVal;
       }
 
-      public ICollection getConstructorSigns(string primitiveName)
-      {
+      public ICollection getConstructorSigns(string primitiveName) {
          ArrayList retVal = new ArrayList();
          Log.assert(myTypeLookupTable[primitiveName] != null);
          Type t = myTypeLookupTable[primitiveName] as Type;
-         foreach (ConstructorInfo ci in t.GetConstructors())
-         {
+         foreach (ConstructorInfo ci in t.GetConstructors()) {
             string sign = "";
             sign = t.Name + " (";
             foreach (ParameterInfo pi in ci.GetParameters())
@@ -181,8 +165,7 @@ namespace BCG
          return retVal;
       }
 
-      public string getAssemblyName(string primitiveName)
-      {
+      public string getAssemblyName(string primitiveName) {
          Log.assert(myTypeLookupTable[primitiveName] != null);
          return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
       }

src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs

@@ -42,7 +42,7 @@ namespace BCG
                Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
             if (myStateTable != null)
                value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
-         }
+      }
          remove
          {
             // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
@@ -64,6 +64,7 @@ namespace BCG
 
       public void update(Stream[] streams)
       {
+         bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
          //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
          //data element for that tag in the ViewStreamDatabase
          if (myOnUpdateDelegate != null)

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs

@@ -3,16 +3,15 @@ using System.IO;
 using System.Collections;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
    /// </summary>
-   public class ClientDeserializationController : IClientDeserializer
-   {
+   public class ClientDeserializationController : IClientDeserializer {
       private IClientDeserializer myDeserializer;
-		
-      public ClientDeserializationController () {}
+      private int _n_desers = 0;
+
+      public ClientDeserializationController() { }
 
       #region IClientDeserializer Members
 
@@ -23,42 +22,41 @@ namespace BCG
       /// <param name="streams"></param>
       /// <param name="theOps"></param>
       /// <param name="theRows"></param>
-      public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         if (myDeserializer != null)
-         {
-            myDeserializer.deserialize (streams, out theOps, out theRows);
-         }
-         else 
-         {
-            Stream totalStream = UtilityMethods.getAggregatedStream (streams);
+      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
+         bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
+         if (myDeserializer != null) {
+            myDeserializer.deserialize(streams, out theOps, out theRows);
+         } else {
+            Stream totalStream = UtilityMethods.getAggregatedStream(streams);
             long initialPos = totalStream.Position;
-            BinaryReader br = new BinaryReader (totalStream);
+            BinaryReader br = new BinaryReader(totalStream);
             byte op;
-            try 
-            {
-               op = br.ReadByte ();
-            }
-            catch (EndOfStreamException)
-            {
+            try {
+               op = br.ReadByte();
+            } catch (EndOfStreamException) {
                op = Byte.MaxValue;
             }
-            if (op == StreamOpCode.create)
-            {
-               br.ReadString (); //pubsubclientdeserializer
-               string deserName = br.ReadString ();   
-               myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
+            if (op == StreamOpCode.create) {
+               br.ReadString(); //pubsubclientdeserializer
+               string deserName = br.ReadString();
+               myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
                   as IClientDeserializer;
                totalStream.Position = initialPos;
-               myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
-            }
-            else
-            {
+               myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
+            } else {
                // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
-               theOps = new byte[]{};
-               theRows = new object[]{};
+               theOps = new byte[] { };
+               theRows = new object[] { };
             }
          }
+         var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
+         int i = 0;
+         foreach (var m in msgs) {
+            bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
+         }
+         _n_desers++;
+         // log it here
+
       }
 
 

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs

@@ -5,61 +5,61 @@ using System.IO;
 using BCG.Utilities;
 //using BCG.Utilities.BlueConfiguration;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
    /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) 
    /// </summary>
-   internal class ClientUniversalRowDeserializer : IClientRowDeserializer
-   {
+   internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
       private IBlueTableSchema mySchema;
       private int myNullMapLength = 0;
       private PrimitiveDeserializationHelper myDeserializationHelper;
       private BitArray myNullMap;
       private int[] mySerIndexToSchemaIndexMap;
-		
-      public ClientUniversalRowDeserializer (IBlueTableSchema schema)
-      {
-         myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
-         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
-         myInitReorderSchema ();
 
-         
+      public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
+         myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
+         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
+         myInitReorderSchema();
+
+
       }
 
       #region Schema Reordering initialization
-      private void myInitReorderSchema ()
-      {
-         int idColIndex = mySchema["id"].colNo;
-         int[] temp = new int [mySchema.columnInfo.Length];
-         foreach (ColumnInfo ci in mySchema.columnInfo)
-         {
-            temp[ci.colNo] =
-               ci.colNo == idColIndex ? 0 :
-               ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
+      private void myInitReorderSchema() {
+         try {
+            int idColIndex = mySchema["id"].colNo;
+            int[] temp = new int[mySchema.columnInfo.Length];
+            foreach (ColumnInfo ci in mySchema.columnInfo) {
+               temp[ci.colNo] =
+                  ci.colNo == idColIndex ? 0 :
+                  ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
+            }
+            mySerIndexToSchemaIndexMap = new int[temp.Length];
+            for (int i = 0; i < temp.Length; i++)
+               mySerIndexToSchemaIndexMap[temp[i]] = i;
+         } catch (Exception ex) {
+            string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
+            string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
+            bts.Log.Gen.error(err_msg);
+            throw new ApplicationException(err_msg);
          }
-         mySerIndexToSchemaIndexMap = new int[temp.Length];
-         for (int i = 0; i < temp.Length; i ++)
-            mySerIndexToSchemaIndexMap [temp[i]] = i;
       }
       #endregion
 
       #region IClientRowDeserializer Members
 
-      public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
-      {
-         br.ReadInt32 ();  //length information for the row not needed
-         myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
-         object[] values = new object [mySchema.columnNames.Length];
-         for (int i = 0; i < values.Length; i++)
-         {
+      public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
+         br.ReadInt32();  //length information for the row not needed
+         myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
+         object[] values = new object[mySchema.columnNames.Length];
+         for (int i = 0; i < values.Length; i++) {
             int colNo = mySerIndexToSchemaIndexMap[i];
-            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
+            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
          }
 
 
-             r = new BlueRow(mySchema, values);
+         r = new BlueRow(mySchema, values);
 
       }
 

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs

@@ -1,22 +1,149 @@
 using System;
-using System.Collections;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+
+namespace BCG {
+   public class StreamIOMessage {
+      public enum StreamOp {
+         Create = 0,
+         Add = 1,             //adds a single row
+         Remove = 2,          //removes a single row
+         Change = 3,          //changes a single row
+         MultipleRemoveAdd = 4,//removes and adds multiple rows
+         MultipleChange = 5 //changes multiples rows          
+      }
+      public class CreateMessage {
+         public IBlueRowCollection rows;
+         public override string ToString() {
+            string rv = "Create message ";
+            if (rows == null) {
+               rv += " rows: null";
+               return rv;
+            }
+            foreach (IBlueRow r in rows) {
+               rv += $"id: {r["id"]} ";
+            }
+            return rv;
+         }
+      }
+      public class AddMessage {
+         public IBlueRow added_row;
+      }
+      public class RemoveMessage {
+         public IBlueRow removed_row;
+      }
+      public class ChangeMessage {
+         public IBlueRow changed_row;
+      }
+      public class MultipleRemoveAddMessage {
+         public IBlueRowCollection removed;
+         public IBlueRowCollection added;
+         public override string ToString() {
+            string rv = "MultipleRemoveAdd removed: ";
+            foreach (IBlueRow r in removed) {
+               rv += $"id: {r["id"]} ";
+            }
+            rv += "added: ";
+            foreach (IBlueRow r in added) {
+               rv += $"id: {r["id"]} ";
+            }
+            return rv;
+         }
+      }
+      public class MultipleChangeMessage {
+         public IBlueRowCollection changed;
+      }
+
+      public StreamOp op;
+      public CreateMessage create_message = null;
+      public AddMessage add_message = null;
+      public RemoveMessage remove_message = null;
+      public ChangeMessage change_message = null;
+      public MultipleRemoveAddMessage multiple_remove_add_message = null;
+      public MultipleChangeMessage multiple_change_message = null;
+      public StreamIOMessage(byte op_, object data) {
+         op = (StreamIOMessage.StreamOp)(op_);
+         switch (op) {
+            case StreamIOMessage.StreamOp.Create:
+               IBlueRowCollection state = null;
+               // TODO, not sure why it can be an array and by itself
+               if (data is IBlueRowCollection) state = data as IBlueRowCollection;
+               else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
+               create_message = new CreateMessage() { rows = state };
+               break;
+            case StreamIOMessage.StreamOp.Add:
+               bts.Log.assert(false);
+               break;
+            case StreamIOMessage.StreamOp.Remove:
+               bts.Log.assert(false);
+               break;
+            case StreamIOMessage.StreamOp.Change:
+               bts.Log.assert(false);
+               break;
+            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
+               IBlueRowCollection[] row_args = data as IBlueRowCollection[];
+               multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
+               break;
+            case StreamIOMessage.StreamOp.MultipleChange:
+               //IBlueRowCollection[] row_args = data as IBlueRowCollection[];
+               multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
+               break;
+         }
+      }
+
+
+      public override string ToString() {
+         switch (op) {
+            case StreamIOMessage.StreamOp.Create:
+               return create_message.ToString();
+               break;
+            case StreamIOMessage.StreamOp.Add:
+               bts.Log.assert(false);
+               return String.Empty;
+               break;
+            case StreamIOMessage.StreamOp.Remove:
+               bts.Log.assert(false);
+               return String.Empty;
+               break;
+            case StreamIOMessage.StreamOp.Change:
+               bts.Log.assert(false);
+               return String.Empty;
+               break;
+            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
+               return multiple_remove_add_message.ToString();
+               break;
+            case StreamIOMessage.StreamOp.MultipleChange:
+               return multiple_change_message.ToString();
+               break;
+         }
+         bts.Log.assert(false);
+         return String.Empty;
+      }
+      public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
+         int i = 0;
+         List<StreamIOMessage> rv = new List<StreamIOMessage>();
+         foreach (byte op in ops) {
+            object data = op_data.GetValue(i++);
+            rv.Add(new StreamIOMessage(op, data));
+         }
+         return rv;
+      }
+   }
 
-namespace BCG
-{
    /// <summary>
    /// Performs deserialization inside the client app. The client gets either the complete data set or the
    /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
    /// </summary>
-   public interface IClientDeserializer 
-   {
+   public interface IClientDeserializer {
       /// <summary>
       /// deserializes the data received by the client from the pubsub
       /// </summary>
       /// <param name="serializedData">the data in byte format received from pubsub</param>
       /// <param name="theOps">the operations that the data maps onto</param>
       /// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
-      void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
+      void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
    }
-
 }
+
+

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs

@@ -2,13 +2,11 @@ using System;
 using System.IO;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. 
    /// </summary>
-   public interface IClientRowDeserializer
-   {
-      void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
+   public interface IClientRowDeserializer {
+      void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs

@@ -2,171 +2,158 @@ using System;
 using System.Collections;
 using System.IO;
 
-namespace BCG
-{
-	/// <summary>
-	/// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer 
-	/// that it initializes according to the table, to optimize serialiazation, hence bw and performance
-	/// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
-	/// ie the first call will always be create.
-	/// </summary>
-	public class PubTablePublisher : IPubTablePublisher
-	{
+namespace BCG {
+   /// <summary>
+   /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer 
+   /// that it initializes according to the table, to optimize serialiazation, hence bw and performance
+   /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
+   /// ie the first call will always be create.
+   /// </summary>
+   public class PubTablePublisher : IPubTablePublisher {
+      private readonly string _tag;
       protected IBlueDataSerializer mySerializer;
       protected PublisherCore myPubCore;
-		
-      public PubTablePublisher (PublisherCore publisherCore) 
-      {
+
+      public PubTablePublisher(string tag, PublisherCore publisherCore) {
+         _tag = tag;
+         bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}");
          myPubCore = publisherCore;
-         mySerializer = new BlueDataSerializationController ();
+         mySerializer = new BlueDataSerializationController();
       }
 
       #region IPubTablePublisher Members
 
-      public virtual void addRow (IBlueRow row)
-      {
+      public virtual void addRow(IBlueRow row) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.add, 
-                                 myGetRowCollectionArray (row),
+         mySerializer.serialize(StreamOpCode.add,
+                                 myGetRowCollectionArray(row),
                                  out data);
-         myPublish (data, row.schema.tableName);
+         myPublish(data, row.schema.tableName);
       }
 
-      public virtual void deleteRow (IBlueRow row)
-      {
+      public virtual void deleteRow(IBlueRow row) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.remove, 
-                                 myGetRowCollectionArray (row),
+         mySerializer.serialize(StreamOpCode.remove,
+                                 myGetRowCollectionArray(row),
                                  out data);
-         myPublish (data, row.schema.tableName);
+         myPublish(data, row.schema.tableName);
       }
 
-      public virtual void changeRow(IBlueRow row)
-      {
+      public virtual void changeRow(IBlueRow row) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.change, 
-                                 myGetRowCollectionArray (row),
+         mySerializer.serialize(StreamOpCode.change,
+                                 myGetRowCollectionArray(row),
                                  out data);
-         myPublish (data, row.schema.tableName);
+         myPublish(data, row.schema.tableName);
       }
 
-      public virtual void createTable (IBlueRowCollection rows)
-      {
+      public virtual void createTable(IBlueRowCollection rows) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.create,
-                                 new IBlueRowCollection [] {rows},
+         mySerializer.serialize(StreamOpCode.create,
+                                 new IBlueRowCollection[] { rows },
                                  out data);
-         myPublish (data, rows.schema.tableName);
+         bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
+         myPublish(data, rows.schema.tableName);
+         // byte[] ops;
+         // Array rows_ds;
+         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
       }
 
-      public virtual void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows)
-      {
-         byte[] data;
-         if (myIsRealChange (delRows, addRows))
-            batchChange (addRows);
-         else
-         {
-            mySerializer.serialize (StreamOpCode.multipleRemoveAdd,
-                                    new IBlueRowCollection []{delRows, addRows},
+      //private ClientDeserializationController _test_deserializer = new ClientDeserializationController();
+      public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
+         byte[] data = null;
+         if (myIsRealChange(delRows, addRows))
+            batchChange(addRows);
+         else {
+            mySerializer.serialize(StreamOpCode.multipleRemoveAdd,
+                                    new IBlueRowCollection[] { delRows, addRows },
                                     out data);
-            myPublish (data, delRows.schema.tableName);
+            myPublish(data, delRows.schema.tableName);
          }
+         // byte[] ops;
+         // Array rows_ds;
+         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
       }
 
-      public virtual void batchChange (IBlueRowCollection rows)
-      {
+      public virtual void batchChange(IBlueRowCollection rows) {
          byte[] data;
-         
-         mySerializer.serialize (StreamOpCode.multipleChange,
-                                 new IBlueRowCollection [] {rows},
+
+         mySerializer.serialize(StreamOpCode.multipleChange,
+                                 new IBlueRowCollection[] { rows },
                                  out data);
-         myPublish (data, rows.schema.tableName);
+         myPublish(data, rows.schema.tableName);
       }
 
       #endregion
 
       #region private methods
-      private void myPublish (byte[] data, string tableName)
-      {
+      private void myPublish(byte[] data, string tableName) {
+         bts.Log.Gen.info($"PubTablePublisher.myPublish table: {tableName} data: {data.Length}");
          if (data.Length > 0)
-            myPubCore.publish ("DataStreamTag", 
-                                 PubStreamPrefixer.prefix (tableName),
-                                 new MemoryStream (data, 0, data.Length, false, true));
+            myPubCore.publish("DataStreamTag",
+                                 PubStreamPrefixer.prefix(tableName),
+                                 new MemoryStream(data, 0, data.Length, false, true));
       }
 
-      private IBlueRowCollection[] myGetRowCollectionArray (IBlueRow r)
-      {
-         return new IBlueRowCollection []{
+      private IBlueRowCollection[] myGetRowCollectionArray(IBlueRow r) {
+         return new IBlueRowCollection[]{
                                             BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r})
                                          };
       }
 
-      private bool myIsRealChange (IBlueRowCollection delRows, IBlueRowCollection addRows)
-      {
-         if (delRows.Count == addRows.Count)
-         {
+      private bool myIsRealChange(IBlueRowCollection delRows, IBlueRowCollection addRows) {
+         if (delRows.Count == addRows.Count) {
             foreach (IBlueRow dRow in delRows)
-               if (!addRows.containsID ((int)dRow["id"]))
+               if (!addRows.containsID((int)dRow["id"]))
                   return false;
             return true;
-         }
-         else
+         } else
             return false;
       }
       #endregion
    }
 
-   public class IndependentTablePublisher : PubTablePublisher
-   {
+   public class IndependentTablePublisher : PubTablePublisher {
       bool myIsCreated = false;
 
-      public IndependentTablePublisher (PublisherCore pubCore) : base (pubCore)
-      {}
+      public IndependentTablePublisher(string tag, PublisherCore pubCore) : base(tag, pubCore) { }
 
-      public override void addRow (IBlueRow row)
-      {
-         myCheckCreate (row.schema);
-         base.addRow (row);
+      public override void addRow(IBlueRow row) {
+         myCheckCreate(row.schema);
+         base.addRow(row);
       }
 
-      public override void batchChange (IBlueRowCollection rows)
-      {
-         myCheckCreate (rows.schema);
-         base.batchChange (rows);
+      public override void batchChange(IBlueRowCollection rows) {
+         myCheckCreate(rows.schema);
+         base.batchChange(rows);
       }
 
-      public override void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows)
-      {
-         myCheckCreate (delRows.schema);
-         base.batchRemoveAdd (delRows, addRows);
+      public override void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
+         myCheckCreate(delRows.schema);
+         base.batchRemoveAdd(delRows, addRows);
       }
 
-      public override void changeRow (IBlueRow row)
-      {
-         myCheckCreate (row.schema);
-         base.changeRow (row);
+      public override void changeRow(IBlueRow row) {
+         myCheckCreate(row.schema);
+         base.changeRow(row);
       }
 
-      public override void deleteRow (IBlueRow row)
-      {
-         myCheckCreate (row.schema);
-         base.deleteRow (row);
+      public override void deleteRow(IBlueRow row) {
+         myCheckCreate(row.schema);
+         base.deleteRow(row);
       }
 
-      public override void createTable(IBlueRowCollection rows)
-      {
+      public override void createTable(IBlueRowCollection rows) {
          myIsCreated = true;
-         base.createTable (rows);
+         base.createTable(rows);
       }
 
 
-      private void myCheckCreate (IBlueTableSchema schema)
-      {
-         if (!myIsCreated)
-         {
+      private void myCheckCreate(IBlueTableSchema schema) {
+         if (!myIsCreated) {
             //so that the initiazliation work is done on the serializer
             byte[] placeHolder;
-            mySerializer.serialize (StreamOpCode.create, new IBlueRowCollection [] {new BlueTable (schema)}, out placeHolder);
+            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { new BlueTable(schema) }, out placeHolder);
             myIsCreated = true;
          }
       }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs

@@ -1,18 +1,16 @@
 using System;
 
-namespace BCG
-{
-	/// <summary>
-	/// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
-	/// the table name, from the config, and wraps around it. The external code that will deal with user calls for
-	/// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
-	/// serialization.
-	/// </summary>
-	public class BlueDataSerializationController : IBlueDataSerializer
-	{
+namespace BCG {
+   /// <summary>
+   /// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
+   /// the table name, from the config, and wraps around it. The external code that will deal with user calls for
+   /// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
+   /// serialization.
+   /// </summary>
+   public class BlueDataSerializationController : IBlueDataSerializer {
       private IBlueDataSerializer mySerializer;
 
-		public BlueDataSerializationController () {}
+      public BlueDataSerializationController() { }
 
       #region IBlueDataSerializer Members
 
@@ -23,43 +21,37 @@ namespace BCG
       /// <param name="op"></param>
       /// <param name="rows"></param>
       /// <param name="serializedData"></param>
-      public void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData)
-      {
-         if (op == StreamOpCode.create)
-         {
+      public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) {
+         if (op == StreamOpCode.create) {
             if (mySerializer == null)
                mySerializer = myGetSerializerForTable(rows[0].schema);
-                                                         
-            mySerializer.serialize (op, rows, out serializedData);
-         }
-         else
-         {
-            if (mySerializer != null)
-            {
-               mySerializer.serialize (op, rows, out serializedData);
-            }
-            else
-            {
-               serializedData = new byte[] {};
+
+            mySerializer.serialize(op, rows, out serializedData);
+         } else {
+            if (mySerializer != null) {
+               mySerializer.serialize(op, rows, out serializedData);
+            } else {
+               serializedData = new byte[] { };
             }
          }
       }
       #endregion
 
-      private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema)
-      {
+      private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) {
+         // TODOds
+         return new UniversalSerializer(theSchema);
          string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName);
 
          object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName);
 
-         if(serializerArgs != null)
+         if (serializerArgs != null)
             return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer;
          else
             return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer;
 
-         
+
 
       }
-      
+
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs

@@ -8,7 +8,7 @@ namespace BCG
 	public class UniversalSerializer : BlueDataSerializer
 	{
 		public UniversalSerializer (IBlueTableSchema schema)
-          : base ("BCG.PubsubUniversalDeserializer, DataStreamGateway", "BCG.ClientUniversalDeserializer")
+          : base ("BCG.PubsubUniversalDeserializer, DataStreamCore", "BCG.ClientUniversalDeserializer")
 		{
          myRowSerializer = new UniversalRowSerializer (schema);
 		}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs

@@ -7,10 +7,11 @@ using System.Text;
 namespace BCG
 {
    /// <summary>
-   /// Base class for all deserializers. 
+   /// Base class for all pubsubs deserializers. 
    /// </summary>
 	public abstract class PubsubDeserializer : IPubsubDeserializer
 	{
+      // this is where the stream state is stored in pubsub?
       protected IDictionary myState = new Hashtable ();
       protected IBlueTableSchema mySchema;
       protected string myClientDeserializerName;   //for the getState method
@@ -35,6 +36,7 @@ namespace BCG
             }
             catch (EndOfStreamException) 
             {
+               // TODO, logically break loop
                break;
             }
             if (op == StreamOpCode.create)

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs

@@ -2,24 +2,22 @@ using System;
 using System.Collections;
 using System.IO;
 
-namespace BCG
-{
-	public class PubsubUniversalDeserializer : PubsubDeserializer
-	{
-		public PubsubUniversalDeserializer() {}
+namespace BCG {
+   public class PubsubUniversalDeserializer : PubsubDeserializer {
+      public PubsubUniversalDeserializer() {
+         bts.Log.Gen.info("PubsubUniversalDeserializer.cons");
+      }
 
-      protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
-      {
-         myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myDeserializer;
-         Hashtable retVal = new Hashtable ();
-         for (int i = 0; i < noOfRows; i++)
-         {
+      protected override IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) {
+         myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
+         Hashtable retVal = new Hashtable();
+         for (int i = 0; i < noOfRows; i++) {
             int id; object row;
-            myDeserializer.deserializeRow (myState, br, op, out id, out row);
-            retVal.Add (id, row);
+            myDeserializer.deserializeRow(myState, br, op, out id, out row);
+            retVal.Add(id, row);
          }
          return retVal;
       }
 
-	}
+   }
 }

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs

@@ -1,54 +1,46 @@
 using System;
 using System.IO;
-using System.Collections;
+using System.Collections.Generic;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Keeps a collection of TagStateBanks to hold the state of each tag
    /// </summary>
-	public class PubsubServerTagHandler : IPubSubServerTagHandler
-	{
-      private IDictionary myTagStates = new Hashtable ();
+	public class PubsubServerTagHandler : IPubSubServerTagHandler {
+      private IDictionary<string, TagStateBank> _tag_state_map = new Dictionary<string, TagStateBank>();
       private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats;
          false;
-      
-      public PubsubServerTagHandler() {}
-     
+
       #region IPubSubServerTagHandler Members
 
-      public byte[] getContent (string tag)
-      {
-         TagStateBank stateBank = myTagStates[tag] as TagStateBank;
-         if (stateBank != null)
-         {
-            return stateBank.getConent ();
-         }
-         else
-         {
-            // BlueEventLog.logEvent (String.Format ("Information requested for tag {0} which does not exist now", tag), BlueEventLevel.Warning);
-            return new byte[] {};
+      public byte[] getContent(string tag) {
+         TagStateBank tag_state_bank;
+         if (_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
+            return tag_state_bank.getConent();
+         } else {
+            bts.Log.Gen.warn("Information requested for tag: {tag}, it does not exist now");
+            return new byte[] { };
          }
       }
 
-      public void update (string tag, Stream[] messageBodies)
-      {
-         if (myPublishStatistics)
-         {
+      public void update(string tag, Stream[] messageBodies) {
+         bts.Log.Gen.info($"PubsubServerTagHandler.update tag: {tag} streams: {messageBodies.Length}");
+         if (myPublishStatistics) {
             StatisticsLogger.instance.log(tag, messageBodies);
          }
-         if (!myTagStates.Contains (tag))
-         {
-            // BlueEventLog.logEvent (String.Format ("Tag {0} received for the first time and its state created", tag), BlueEventLevel.Information);
-            myTagStates.Add (tag, new TagStateBank ());
+         TagStateBank tag_state_bank;
+         if (!_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
+            bts.Log.Gen.info("tag: {tag} received for the first time creating TagStateBank for it now");
+            tag_state_bank = new TagStateBank();
+            _tag_state_map.Add(tag, tag_state_bank);
          }
-         (myTagStates[tag] as TagStateBank).update (messageBodies);
+         tag_state_bank.update(messageBodies);
       }
 
-      public void doWork() {}
+      public void doWork() { }
 
-      public string[] supportedTagTypes {get {return new string [] {"DataStreamTag"};}}
+      public string[] supportedTagTypes { get { return new string[] { "DataStreamTag" }; } }
 
       public void register(InternalPublishDelegate callback) { }
 

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs

@@ -1,26 +1,26 @@
 using System;
-using System.Collections;
+using System.Collections.Generic;
 using System.IO;
 
-namespace BCG
-{
-   public class TagStateBank
-   {
-      //private IDictionary myState = new Hashtable ();
-      private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController ();
-		
-      public TagStateBank () {}
+namespace BCG {
+   public class TagStateBank {
+      private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController();
 
-      public byte[] getConent ()
-      {
-         return myDeserializationController.getState ();
+      public TagStateBank() { }
+
+      public TagStateBank(string tag) {
+         bts.Log.Gen.info("TagStateBank created for tag: {tag}");
       }
 
-      public void update (Stream[] streams)
-      {
+      public byte[] getConent() {
+         return myDeserializationController.getState();
+      }
+
+      public void update(Stream[] streams) {
+         bts.Log.Gen.info($"TagStateBank.update streams: {streams.Length}");
          byte[] ops;
          Array rowCollection;
-         myDeserializationController.deserialize (streams, out ops, out rowCollection);
+         myDeserializationController.deserialize(streams, out ops, out rowCollection);
       }
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs

@@ -9,7 +9,7 @@ namespace BCG
    public class CompressionSerializer : IBlueDataSerializer
    {
       private IBlueRowSerializer myRowSerializer;
-      private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamGateway", myClientDeserializerName = "BCG.CompressionClientDeserializer";
+      private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamCore", myClientDeserializerName = "BCG.CompressionClientDeserializer";
       
       public CompressionSerializer (IBlueTableSchema schema)
       {

src/DataStreamSuite/DataStreamCore/Messaging/Test/CompressionSerializationTest.cs

@@ -0,0 +1,236 @@
+// using System;
+// using System.IO;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+
+//    /// <summary>
+//    /// Tests the deserialization happening within the pubsub.
+//    /// </summary>
+//    [TestGroup]
+//    public class CompressionSerializationTest
+//    {
+//       protected IBlueTableSchema mySchema;
+//       protected BlueTable myTable;
+//       protected IBlueDataSerializer mySer;
+//       protected IPubsubDeserializer myPubsubDeser;
+//       protected IClientDeserializer myClientDeser;
+
+
+//       [Test]
+//       public static void test_compression()
+//       {
+//          int n_len = 10000000;
+//          for (int t = 0; t < 10; t++)
+//          {
+//             Random rnd = new Random(DateTime.Now.Millisecond);
+//             byte[] data = new byte[n_len];
+//             rnd.NextBytes(data);
+
+//             BCG.Utilities.General.HighPerformanceTimer timer = new Utilities.General.HighPerformanceTimer();
+//             timer.start();
+//             byte[] compr = UtilityMethods.compress(new MemoryStream(data));
+//             timer.stop();
+//             Console.WriteLine("Time to compress {0} bytes {1} msecs ratio {2}", n_len, timer.lastIntervalTime, (1.0 * data.Length)/(1.0 * compr.Length));
+
+//             timer.start();
+//             Stream decompr_str = UtilityMethods.decompress(compr);
+//             timer.stop();            
+//             Console.WriteLine("Time to de-compress {0} bytes {1} msecs", n_len, timer.lastIntervalTime);
+
+//             decompr_str.Position = 0;
+//             Assert.AreEqual(data.Length, (int)decompr_str.Length);
+//             for (int i = 0; i < data.Length; i++)
+//                Assert.AreEqual(data[i], decompr_str.ReadByte());
+//          }
+//       }
+
+//       public CompressionSerializationTest()
+//       {
+//          mySchema = new BlueTableSchema(myGetTableName(),
+//                                           new ColumnInfo("id", 0, typeof(int)),
+//                                           new ColumnInfo("i1", 1, typeof(int)),
+//                                           new ColumnInfo("d1", 2, typeof(double)),
+//                                           new ColumnInfo("dt", 3, typeof(DateTime)),
+//                                           new ColumnInfo("x", 4, typeof(byte))
+//             );
+//          mySer = new CompressionSerializer(mySchema);
+//          myClientDeser = new ClientDeserializationController();
+//          myPubsubDeser = new PubsubDeserializationController();
+//       }
+
+//       protected virtual string myGetTableName()
+//       {
+//          return "testUniversal";
+//       }
+
+//       /// <summary>
+//       /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
+//       /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer. 
+//       /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
+//       /// through the clientDeserializer to get all the rows.
+//       /// </summary>
+//       [Test]
+//       public void testEntireProcess()
+//       {
+//          myTable = new BlueTable(mySchema);
+//          byte[] data;
+//          int id;
+//          //create
+//          for (id = 0; id < 100; id++)
+//             myTable.addRow(myGetRow(id));
+
+//          byte[] ops;
+//          Array rows;
+//          IBlueRowCollection del, add;
+//          IBlueRowCollection[] receivedData;
+//          IBlueRow r;
+
+//          //create
+//          mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
+//          myWrite(StreamOpCode.create, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.create, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+
+//          BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //single add
+//          r = myGetRow(id);
+//          BlueTable temp = new BlueTable(mySchema);
+//          myTable.addRow(r);
+//          temp.addRow(r);
+//          mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
+//          myWrite(StreamOpCode.add, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.add, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //single delete
+//          r = myTable[0];   //delete row with id 0
+//          temp = new BlueTable(mySchema);
+//          myTable.deleteRow(0);
+//          temp.addRow(r);
+//          mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
+//          myWrite(StreamOpCode.remove, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.remove, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //batch deletes and adds
+//          //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
+//          BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
+//          for (int i = 1; i < 25; i++)
+//          {
+//             delRows.addRow(myTable[i]);
+//             myTable.deleteRow(i);
+//          }
+//          for (id = 101; id < 125; id++)
+//          {
+//             r = myGetRow(id);
+//             addRows.addRow(r);
+//             myTable.addRow(r);
+//          }
+//          for (int i = 25; i < 125; i++)
+//             Assert.AreEqual(true, myTable.containsID(i));
+//          Assert.AreEqual(100, myTable.Count);
+
+//          mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
+//          myWrite(StreamOpCode.multipleRemoveAdd, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(2, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+//          BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //pure changes only
+//          //change the first 25, 25-49
+//          BlueTable changedRows = new BlueTable(mySchema);
+//          for (int i = 25; i < 50; i++)
+//          {
+//             MutableBlueRow mr = new MutableBlueRow(myTable[i]);
+//             mr.setValue(4, Byte.MaxValue);
+//             myTable.changeRow(mr);
+//             changedRows.addRow(mr);
+//          }
+//          mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
+//          myWrite(StreamOpCode.multipleChange, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //final test
+//          data = myPubsubDeser.getState();
+//          new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.create, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//       }
+
+//       private IBlueRow myGetRow(int id)
+//       {
+//          return new BlueRow(mySchema,                        // the day pp fixed this test. 
+//                               id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
+//       }
+
+//       private void myWrite(byte op, byte[] data)
+//       {
+//          string s = "";
+//          if (op == StreamOpCode.add)
+//             s = "add one row";
+//          else if (op == StreamOpCode.change)
+//             s = "change one row";
+//          else if (op == StreamOpCode.create)
+//             s = "create";
+//          else if (op == StreamOpCode.multipleChange)
+//             s = "multiple change";
+//          else if (op == StreamOpCode.multipleRemoveAdd)
+//             s = "multiple add remove";
+//          else if (op == StreamOpCode.remove)
+//             s = "remove one row";
+//          Console.WriteLine("op = {0}, size = {1}", s, data.Length);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingRowSerializationTest.cs

@@ -0,0 +1,100 @@
+// using System;
+// using System.IO;
+// using System.Collections;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+//    [TestGroup]
+// 	public class FixedChangingRowSerializationTest
+// 	{
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
+//                                                                new ColumnInfo ("i", 0, typeof(int)),
+//                                                                new ColumnInfo ("b", 1, typeof(bool)),
+//                                                                new ColumnInfo ("d", 2, typeof(double)),
+//                                                                new ColumnInfo ("id", 3, typeof(int)),
+//                                                                new ColumnInfo ("dt", 4, typeof(DateTime)),
+//                                                                new ColumnInfo ("bt", 5, typeof(byte)));
+//       private BlueTable myState;
+//       private Random myRnd = new Random (DateTime.Now.Millisecond);
+//       private IBlueRowSerializer mySerializer;
+//       private IClientRowDeserializer myRowDeserializer;
+
+// 		public FixedChangingRowSerializationTest ()
+// 		{
+//          myState = new BlueTable (mySchema);  
+//          mySerializer = new FixedChangingRowSerializer (mySchema, "bt");
+//          myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, new string[] {"bt"});
+// 		}
+
+//       [Test]
+//       public void addTest ()
+//       {
+//          for (int i = 0; i < 1000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow r = myCreateRow (i);
+//             mySerializer.serializeRow (r, StreamOpCode.add, bw);   
+            
+//             BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
+//             Assert.AreEqual (r["i"], br.ReadInt32());
+//             Assert.AreEqual (r["b"], br.ReadBoolean ());
+//             Assert.AreEqual (r["d"], br.ReadDouble ());
+//             Assert.AreEqual (r["id"], br.ReadInt32 ());
+//             Assert.AreEqual (((DateTime)r["dt"]).Ticks, br.ReadInt64());
+//             Assert.AreEqual (r["bt"], br.ReadByte ());
+//          }
+//       }
+
+//       [Test]
+//       public void removeTest ()
+//       {
+//          for (int i = 0; i < 1000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow r = myCreateRow (i);
+//             mySerializer.serializeRow (r, StreamOpCode.remove, bw);   
+            
+//             BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
+//             Assert.AreEqual (r["id"], br.ReadInt32());
+//          }
+//       }
+
+//       [Test]
+//       public void changeTest ()
+//       {
+//          for (int i = 0; i < 1000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow r = myCreateRow (i);
+//             mySerializer.serializeRow (r, StreamOpCode.change, bw);   
+            
+//             BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
+//             Assert.AreEqual (r["id"], br.ReadInt32());
+//             Assert.AreEqual (r["bt"], br.ReadByte());
+//             bool isEnd = false;
+//             try
+//             {
+//                br.ReadByte ();
+//             }
+//             catch (EndOfStreamException)
+//             {
+//                isEnd = true;
+//             }
+//             Assert.That (isEnd);
+//          }
+//       }
+      
+//       private IBlueRow myCreateRow (int id)
+//       {
+//          return new BlueRow (mySchema,
+//                               myRnd.Next (1000),
+//                               myRnd.Next (1000) % 2 == 0,
+//                               myRnd.NextDouble (),
+//                               id,
+//                               DateTime.Now,
+//                               Convert.ToByte (10));
+//       }
+
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingSerializationTest.cs

@@ -0,0 +1,40 @@
+// using System;
+
+// namespace BCG
+// {
+
+//    public class TestFixedChangingSerializer : FixedChangingSerializer
+//    {
+//       public TestFixedChangingSerializer (IBlueTableSchema schema)
+//          : base (schema, 
+//                   new string[] {"x"}, 
+//                   "BCG.PubsubTestFixedChangingDeserializer",
+//                   "BCG.ClientTestFixedChangingDeserializer"
+//                 )
+//       {}                                              
+//    }
+
+//    public class PubsubTestFixedChangingDeserializer : PubsubFixedChangingDeserializer
+//    {
+//       public PubsubTestFixedChangingDeserializer () : base (new string[] {"x"})
+//       {}
+//    }
+
+//    public class ClientTestFixedChangingDeserializer : ClientFixedChangingDeserializer
+//    {
+//       public ClientTestFixedChangingDeserializer () : base (new string[] {"x"})
+//       {}
+//    }
+
+
+// 	public class FixedChangingSerializationTest : UniversalSerializationTest
+// 	{
+// 		public FixedChangingSerializationTest () {}
+
+//       protected override string myGetTableName()
+//       {
+//          return "testFixedChanging";
+//       }
+
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/MarketOptionTheoSerializationTest.cs

@@ -0,0 +1,73 @@
+// using System;
+// using System.Collections;
+// using System.IO;
+// using NUnit.Framework;
+// using BCG.Utilities.General;
+
+// namespace BCG
+// {
+//    [TestFixture]
+// 	public class MarketOptionTheoSerializationTest
+// 	{
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("marketOptionTheo",
+//                                                                   new ColumnInfo ("arrayIndex", 0, typeof(byte)),
+//                                                                   new ColumnInfo ("cell", 1, typeof(byte)),
+//                                                                   new ColumnInfo ("expiration", 2, typeof(DateTime)),
+//                                                                   new ColumnInfo ("id", 3, typeof(int)),
+//                                                                   new ColumnInfo ("value", 4, typeof(double))
+//                                                                );
+                                          
+// 		public MarketOptionTheoSerializationTest () {}
+
+//       [Test]
+//       public void serializationPerformanceTest ()
+//       {
+//          MarketOptionTheoSerializer ser = new MarketOptionTheoSerializer (mySchema);
+//          IClientDeserializer deser = new MarketOptionTheoClientDeserializer ();
+//          BlueTable bt = new BlueTable (mySchema);
+//          int count = 3000;
+//          byte[] ops;
+//          Array rows;
+//          HighPerformanceTimer timer = new HighPerformanceTimer ();
+//          byte[] serData = null;
+
+//          for (int i = 0; i < count; i++)
+//             bt.addRow (myGetRow (i));
+
+         
+//          ///serialization
+//          //create
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out serData);
+//          deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
+//          ArrayList times = new ArrayList ();
+//          for (int i = 0; i < 25; i++)
+//          {
+//             timer.start ();
+//             ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {bt, bt}, out serData);
+//             timer.stop ();
+//             times.Add (timer.lastIntervalTime);
+//             Console.WriteLine (String.Format ("serialization of {0} rows, time = {1}, size = {2}", count, timer.lastIntervalTime, serData.Length));
+//          }
+//          times.Sort ();
+//          Console.WriteLine ("Median time = {0}", times[times.Count/2]);
+//          //deserialization
+//          times.Clear ();
+//          for (int i = 0; i < 25; i++)
+//          {
+//             timer.start ();
+//             deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
+//             timer.stop();
+//             times.Add (timer.lastIntervalTime);
+//             Console.WriteLine (String.Format ("deserialization of {0} rows, time = {1}", count, timer.lastIntervalTime));
+//          }
+//          times.Sort ();
+//          Console.WriteLine ("Median time = {0}", times[times.Count/2]);
+//       }
+
+//       private IBlueRow myGetRow (int id)
+//       {
+//          return new BlueRow (mySchema,
+//                              Convert.ToByte (2), Convert.ToByte (4), DateTime.Now, id, 10.324555);
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/Performance.cs

@@ -0,0 +1,46 @@
+// using System;
+// using NUnit.Framework;
+// using BCG.Utilities.General;
+
+// namespace BCG
+// {
+//    [TestFixture]
+// 	public class Performance
+// 	{
+// 		public Performance()
+// 		{
+// 		}
+
+//       [Test]
+//       public void testContructionTime ()
+//       {
+//          double totalTime;
+//          string deserName = "BCG.ClientUniversalDeserializer";
+//          HighPerformanceTimer timer = new HighPerformanceTimer ();
+//          IClientDeserializer cd;
+//          for (int i = 0; i < 100; i++)
+//          {
+//             timer.start ();
+//             cd = Activator.CreateInstance (Type.GetType (deserName)) as IClientDeserializer;
+//             timer.stop ();
+//             //Console.Write (timer.lastIntervalTime.ToString() + " ");
+//          }
+//          Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
+//          totalTime = timer.totalMilliseconds;
+
+//          timer = new HighPerformanceTimer ();
+//          for (int i = 0; i < 100; i++)
+//          {
+//             timer.start ();
+//             cd = new ClientUniversalDeserializer ();
+//             timer.stop ();
+//             //Console.Write (timer.lastIntervalTime.ToString () + " ");
+//          }
+//          Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
+//          Console.WriteLine ("Ratio = " + totalTime/timer.totalMilliseconds);
+//       }
+
+
+
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/TableAccumulatorTest.cs

@@ -0,0 +1,275 @@
+// using System;
+// using System.IO;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+// 	[TestFixture]
+// 	public class TableAccumulatorTest
+// 	{
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("test",
+//                                                                new ColumnInfo ("id", 0, typeof(int)),
+//                                                                new ColumnInfo ("x", 1, typeof(int)));
+//       private SubscriberCore mySubCore = new SubscriberCore ();
+//       private IBlueRowCollection myNr, myAr, myDr;
+		
+//       public TableAccumulatorTest() {}
+
+//       [SetUp]
+//       public void tearDown () 
+//       {
+//          myNr = myAr = myDr = null;
+//       }
+
+//       #region create
+//       [Test]
+//       public void testCreate ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //testing the data
+//          IBlueRowCollection dr, ar;
+//          //new rows
+//          BlueDataUtility.computeDifferences (bt, myNr, out dr, out ar);
+//          Assert.AreEqual (0, dr.Count);
+//          Assert.AreEqual (0, ar.Count);
+//          //added rows
+//          BlueDataUtility.computeDifferences (bt, myAr, out dr, out ar);
+//          Assert.AreEqual (0, dr.Count);
+//          Assert.AreEqual (0, ar.Count);
+//          //deleted rows;
+//          Assert.AreEqual (0, myDr.Count);
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region single add, delete and change
+//       [Test]
+//       public void testAddRemoveChange ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //add
+//          //add a row with id 100
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 100, 200));
+//          ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //testing the add data
+//          Assert.AreEqual (101, myNr.Count);
+//          for (int i = 0; i < 101; i++)
+//             myNr.containsID (i);
+//          Assert.AreEqual (0, myDr.Count);
+//          Assert.AreEqual (1, myAr.Count);
+//          Assert.That (myAr.containsID (100));
+
+//          //remove
+//          //remove the row with id 0
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 0, 0));
+//          ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //testing remove data
+//          Assert.AreEqual (100, myNr.Count);
+//          for (int i = 0; i < 101; i++)
+//          {
+//             Assert.That ((i == 0 && !myNr.containsID (i)) || (i != 0 && myNr.containsID (i)));
+//          }
+//          Assert.AreEqual (0, myAr.Count);
+//          Assert.AreEqual (1, myDr.Count);
+//          Assert.That (myDr.containsID (0));
+         
+//          //change
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 100, -100));
+//          ser.serialize (StreamOpCode.change, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //testing changed data
+//          Assert.AreEqual (100, myNr.Count);
+//          Assert.AreEqual (1, myAr.Count);
+//          Assert.AreEqual (100, BlueDataUtility.extractRow (myAr, 0)["id"]);
+//          Assert.AreEqual (-100, BlueDataUtility.extractRow (myAr, 0)["x"]);
+//          Assert.AreEqual (1, myDr.Count);
+//          Assert.AreEqual (100, BlueDataUtility.extractRow (myDr, 0)["id"]);
+//          Assert.AreEqual (200, BlueDataUtility.extractRow (myDr, 0)["x"]);
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region multiple change
+//       public void testMultipleChange ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //partial rows changed
+//          bt.clear ();
+//          for (int i = 0; i < 50; i++)
+//             bt.addRow (new BlueRow (mySchema, i, -2*i));
+//          ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //checking received data
+//          Assert.AreEqual (100, myNr.Count);
+//          foreach (IBlueRow r in myNr)
+//          {
+//             if ((int)r["id"] < 50)
+//                Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//             else
+//                Assert.AreEqual ((int)r["id"] * 2, r["x"]);
+//          }
+//          Assert.AreEqual (50, myAr.Count);
+//          foreach (IBlueRow r in myAr)
+//             Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//          Assert.AreEqual (50, myDr.Count);
+//          foreach (IBlueRow r in myDr)
+//             Assert.AreEqual ((int)r["id"] * 2, r["x"]);
+
+//          //full rows changed
+//          bt.clear ();
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, -2*i));
+//          ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //checking received data
+//          Assert.AreEqual (100, myNr.Count);
+//          foreach (IBlueRow r in myNr)
+//          {
+//             Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//          }
+//          Assert.AreEqual (100, myAr.Count);
+//          foreach (IBlueRow r in myAr)
+//             Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//          Assert.AreEqual (100, myDr.Count);
+//          foreach (IBlueRow r in myDr)
+//          {
+//             if ((int)r["id"] < 50)
+//                Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//             else
+//                Assert.AreEqual ((int)r["id"] * 2, r["x"]);
+//          }
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region multiple deletes/adds
+//       public void testMultipleDeletesAdds ()
+//       {
+         
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          BlueTable at = new BlueTable (mySchema),
+//                      dt = new BlueTable (mySchema);
+//          for (int i = 0; i < 50; i++)
+//             dt.addRow (new BlueRow (mySchema, i, 2*i));
+//          for (int i = 100; i < 150; i++)
+//             at.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dt, at}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //checking state
+//          Assert.AreEqual (100, myNr.Count);
+//          for (int i = 50; i < 150; i++)
+//             Assert.That (myNr.containsID (i));
+//          Assert.AreEqual (50, myDr.Count);
+//          for (int i = 0; i < 50; i++)
+//             Assert.That (myDr.containsID (i));
+//          Assert.AreEqual (50, myAr.Count);
+//          for (int i = 100; i < 150; i++)
+//             Assert.That (myAr.containsID (i));
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region multiple ops in one stream
+//       [Test]
+//       public void testMultipleOps ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          MemoryStream ms = new MemoryStream ();
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //ms.Write (data, 0, data.Length);
+//          //changing the first 5 rows
+//          bt.clear ();
+//          for (int i = 0; i < 5; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 1000));
+//          ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          //delete id 5
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 5, 10));
+//          ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          //add id 100
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 100, 200));
+//          ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          //delete 4 and add 5
+//          bt.clear ();
+//          BlueTable dels = new BlueTable (bt.schema);
+//          for (int i = 90; i < 94; i++)
+//             dels.addRow (new BlueRow (mySchema, i, i*2));
+         
+//          for (int i = 101; i < 106; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dels, bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          ms.Position = 0;
+//          ta.update (new Stream[] {ms});
+         
+//          Assert.AreEqual (101, myNr.Count);
+//          Assert.AreEqual (10, myDr.Count);
+//          Assert.AreEqual (11, myAr.Count);
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+         
+//       }
+//       #endregion
+
+//       private void myHandleUpdate (IBlueRowCollection nr, IBlueRowCollection ar, IBlueRowCollection dr)
+//       {
+//          myNr = nr;
+//          myAr = ar;
+//          myDr = dr;
+//       }
+
+      
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/UniversalRowSerializerTest.cs

@@ -0,0 +1,58 @@
+// using System;
+// using System.IO;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+//    [TestFixture]
+// 	public class UniversalRowSerializerTest
+// 	{
+//       private IBlueRowSerializer myRowSerializer;
+//       private IClientRowDeserializer myRowDeserializer;
+//       private Random myRnd = new Random (DateTime.Now.Millisecond);
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
+//                                                                new ColumnInfo ("id", 0, typeof(int)),
+//                                                                new ColumnInfo ("int1", 1, typeof(int)),
+//                                                                new ColumnInfo ("int2", 2, typeof(int)),
+//                                                                new ColumnInfo ("double1", 3, typeof(double)),
+//                                                                new ColumnInfo ("double2", 4, typeof(double)),
+//                                                                new ColumnInfo ("byte1", 5, typeof(byte)),
+//                                                                new ColumnInfo ("string1", 6, typeof(string)),
+//                                                                new ColumnInfo ("string2", 7, typeof(string)));
+// 		public UniversalRowSerializerTest () {}
+
+//       [Test]
+//       public void testSerialiazationCycle ()
+//       {
+//          myRowSerializer = new UniversalRowSerializer (mySchema);
+//          myRowDeserializer = new ClientUniversalRowDeserializer (mySchema);
+//          for (int i = 0; i < 10000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow ir = myGetRandomRow ();
+//             myRowSerializer.serializeRow (ir, StreamOpCode.add, bw);
+
+//             IBlueRow or;
+//             (bw.BaseStream as MemoryStream).Position = 0;
+//             BinaryReader br = new BinaryReader (bw.BaseStream);
+//             myRowDeserializer.deserializeRow (null, br, StreamOpCode.add, out or);
+//             Assert.That (BlueDataUtility.computeRowEquality (ir, or));
+//          }
+//       }
+
+//       private IBlueRow myGetRandomRow ()
+//       {
+//          return new BlueRow (mySchema,
+//                                     myRnd.Next (1000),
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.Next (1000) as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object: myRnd.Next (1000) as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : Convert.ToByte (76) as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "abcdefgh" as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object :  "1234567890" as object
+//                                  ); 
+                                    
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/stream_serialization.txt

@@ -0,0 +1,13 @@
+PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
+PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd)
+PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[]
+BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now)
+
+IBlueDataSerializer has just one function
+void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
+the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add
+(BlueDataSerializer.serialize's comment has the serialization layout)
+
+For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams)
+For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer
+

src/DataStreamSuite/DataStreamCore/Misc/IPubSubRouter.cs

@@ -1,20 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace BCG
-{
-   public delegate void PubSubTagURLChangeDelegate(List<string> tagsMoving, string newURL);
-   
-   public interface IPubSubRouter
-   {
-
-      event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
-      event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
-      List<string> getAllPubSubSubURLs();
-      List<string> getAllPubSubPubURLs();
-      string getSubURLForTag(string tag);
-      string getPubURLForTag(string tag);
-   }
-}

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionGrammar.y

@@ -23,6 +23,7 @@
 
 %token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, CS_STRING_CONSTANT, NULL_CONSTANT
 %token PARENTHESIS_OPEN, PARENTHESIS_CLOSE, COMMA, IDENTIFIER, EQUALS, QUOTE_OPEN, QUOTE_CLOSE
+%token LINE_COMMENT
 
 %nonassoc EQUALS
 
@@ -49,6 +50,12 @@ Statement :
    | IDENTIFIER EQUALS Statement {
       $$ = new Assignment($1.Identifier, $3);
    }
+   | LINE_COMMENT Statement {
+      $$ = $2;
+   }
+   | Statement LINE_COMMENT {
+      $$ = $1;
+   }
    ;
 
 Arguments :

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -26,6 +26,9 @@ Comma                    ,
 QuoteOpen                \[
 QuoteClose               \]
 
+/* Comments */
+LineComment \/\/[^\n\r]*(\n|\r|\r\n)
+
 %%
 
 {Equals} {
@@ -92,4 +95,8 @@ QuoteClose               \]
    return (int)DataStreamExpressionToken.IDENTIFIER;
 }
 
+{LineComment} {
+   return (int)DataStreamExpressionToken.LINE_COMMENT;
+}
+
 %%

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -48,14 +48,26 @@ public class PrimitiveExpression : StatementExpression {
       return PrimitiveManager.instance.build(myName, resolvedArgs);
    }
 
+   string myPrintSingleArg(int atColumn, object? o, IList<string> activeBindings) {
+      /// Unwrap <see cref="Argument"/> to get its underlying value
+      while (o is Argument arg) {
+         o = arg.Obj;
+         // Safeguard against infinite loop
+         if (ReferenceEquals(o, arg)) {
+            throw new ArgumentException(
+               $"Detected circular reference in argument " +
+               $"('{nameof(arg)}.{nameof(arg.Obj)}' is '{nameof(arg)}' itself)");
+         }
+      }
 
-   string myPrintSingleArg(int atColumn, object o, IList<string> activeBindings) {
+      string result = new string(' ', atColumn);
+      if (o is null) {
+         return result;
+      }
       if (o is StatementExpression)
          return new string(' ', atColumn) +
             ((o as StatementExpression)?.print(atColumn, activeBindings) ?? string.Empty);
       else {
-         string result = new string(' ', atColumn);
-
          if (o is string)
             result += makeCSString((string)o);
          else if (o is DateTime)

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -3,6 +3,10 @@
   <ItemGroup>
     <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\PubSub\PubSub.csproj" />
   </ItemGroup>
 
   <PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -1,16 +1,84 @@
-using bts.utils;
-
+using System;
 using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
+using Bts.DataStreamExpressions;
+using System.Collections.Generic;
 
-using System;
-using bts;
-using mbts;
+using BCG;
 
-BTSNativeMethods.provide_command_line(args, args.Length);
-// Must initialize config before using log system
-Console.WriteLine("Initializing config...");
-Configuration.initialize();
-for (int i = 0; i < 10; i++)
+
+bts.Log.Gen.info($"Starting DSPublisher");
+Console.WriteLine("Starting DSPublisher...");
+static void init(string[] args)
 {
-   bts.Log.Gen.info($"Starting DataStreamPublisher...log: {i}");
+   Console.WriteLine("Hit key to continue");
+   Console.ReadLine();
+   Func<string, string, Task> login = async (l, p) =>
+   {
+      var login = l;
+      var passwd = p;
+      var service_ctx = bts.api.core.ServiceContext.create();
+      await service_ctx.login(login, passwd);
+   };
+   BTSNativeMethods.provide_command_line(args, args.Length);
+   Configuration.initialize();
+   Task.Run(async () => await login("bts", String.Empty));
 }
+
+init(args);
+_publish_streams();
+
+static ViewStream build_stream(string stream_expression)
+{
+   try
+   {
+      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
+      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
+      return rv;
+
+   }
+   catch (Exception ex)
+   {
+      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+      return null;
+   }
+}
+
+
+void _publish_streams()
+{
+   PubTableBroker pb_broker = new PubTableBroker();
+   CloudStreamManager csm = new CloudStreamManager();
+   // get it from config
+   Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
+        {"bbo","bbo()"},
+        {"instruments","instruments()"},
+        //{"timesales","timesales(10000)"},
+        {"books","books()"},
+      // {"seconds_1_10","second(1,10)"},
+        {"literal_420","literal(\"x\", \"420.0\")"},
+      //  {"bbo_ts","timeSeries(bbo())"}
+   };
+   foreach (var kv in streams_to_publish)
+   {
+      var name = kv.Key;
+      var exp = kv.Value;
+      BCG.ViewStream vs = build_stream(exp);
+      ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
+      vs.onChange += on_change;
+   }
+   while (true)
+   {
+      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
+      ViewStreamDatabase.instance.doWork();
+      csm.doWork();
+   }
+}
+
+
+
+void on_change(object obj)
+{
+   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -12,11 +12,13 @@
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\BlueRegistry\BlueRegistry.csproj" />
     <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
     <ProjectReference Include="..\GuiUtilities\GuiUtilities.csproj" />
     <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
     <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
   </ItemGroup>
 
 </Project>

src/DataStreamSuite/DataStreamQueryAnalyzer/HScrollGrid.resx

@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="utf-8"?>
+<root>
+  <!-- 
+    Microsoft ResX Schema 
+    
+    Version 2.0
+    
+    The primary goals of this format is to allow a simple XML format 
+    that is mostly human readable. The generation and parsing of the 
+    various data types are done through the TypeConverter classes 
+    associated with the data types.
+    
+    Example:
+    
+    ... ado.net/XML headers & schema ...
+    <resheader name="resmimetype">text/microsoft-resx</resheader>
+    <resheader name="version">2.0</resheader>
+    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+        <value>[base64 mime encoded serialized .NET Framework object]</value>
+    </data>
+    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+        <comment>This is a comment</comment>
+    </data>
+                
+    There are any number of "resheader" rows that contain simple 
+    name/value pairs.
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
+    mimetype set.
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
+    extensible. For a given mimetype the value must be set accordingly:
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
+    read any of the formats listed below.
+    
+    mimetype: application/x-microsoft.net.object.binary.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+            : and then encoded with base64 encoding.
+    
+    mimetype: application/x-microsoft.net.object.soap.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+            : and then encoded with base64 encoding.
+
+    mimetype: application/x-microsoft.net.object.bytearray.base64
+    value   : The object must be serialized into a byte array 
+            : using a System.ComponentModel.TypeConverter
+            : and then encoded with base64 encoding.
+    -->
+  <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+    <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+    <xsd:element name="root" msdata:IsDataSet="true">
+      <xsd:complexType>
+        <xsd:choice maxOccurs="unbounded">
+          <xsd:element name="metadata">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" />
+              </xsd:sequence>
+              <xsd:attribute name="name" use="required" type="xsd:string" />
+              <xsd:attribute name="type" type="xsd:string" />
+              <xsd:attribute name="mimetype" type="xsd:string" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="assembly">
+            <xsd:complexType>
+              <xsd:attribute name="alias" type="xsd:string" />
+              <xsd:attribute name="name" type="xsd:string" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="data">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+                <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+              <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+              <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="resheader">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" />
+            </xsd:complexType>
+          </xsd:element>
+        </xsd:choice>
+      </xsd:complexType>
+    </xsd:element>
+  </xsd:schema>
+  <resheader name="resmimetype">
+    <value>text/microsoft-resx</value>
+  </resheader>
+  <resheader name="version">
+    <value>2.0</value>
+  </resheader>
+  <resheader name="reader">
+    <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+  <resheader name="writer">
+    <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+</root>

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.Designer.cs

@@ -5,16 +5,16 @@
       /// <summary>
       /// Clean up any resources being used.
       /// </summary>
-      protected override void Dispose (bool disposing)
+      protected override void Dispose(bool disposing)
       {
          if (disposing)
          {
             if (components != null)
             {
-               components.Dispose ();
+               components.Dispose();
             }
          }
-         base.Dispose (disposing);
+         base.Dispose(disposing);
       }
 
       #region Windows Form Designer generated code
@@ -22,500 +22,549 @@
       /// Required method for Designer support - do not modify
       /// the contents of this method with the code editor.
       /// </summary>
-      private void InitializeComponent ()
+      private void InitializeComponent()
       {
-         this.components = new System.ComponentModel.Container();
+         components = new System.ComponentModel.Container();
+         System.Windows.Forms.DataGridViewCellStyle dataGridViewCellStyle1 = new System.Windows.Forms.DataGridViewCellStyle();
+         System.Windows.Forms.DataGridViewCellStyle dataGridViewCellStyle2 = new System.Windows.Forms.DataGridViewCellStyle();
          System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(MainForm));
-         this.myQueryTextBox = new System.Windows.Forms.RichTextBox();
-         this.myTabControl = new System.Windows.Forms.TabControl();
-         this.myViewTabPage = new System.Windows.Forms.TabPage();
-         this.myDataGrid = new BCG.HScrollGrid();
-         this.myCSharpTabPage = new System.Windows.Forms.TabPage();
-         this.myCSharpText = new System.Windows.Forms.RichTextBox();
-         this.myErrorTabPage = new System.Windows.Forms.TabPage();
-         this.myErrorTextBox = new System.Windows.Forms.RichTextBox();
-         this.myTableListTabPage = new System.Windows.Forms.TabPage();
-         this.myTableListBox = new BCG.Utilities.XPControls.ListBoxXP();
-         this.myTableListTopPanel = new System.Windows.Forms.Panel();
-         this.myTableFilterBox = new System.Windows.Forms.TextBox();
-         this.myTableFilterLabel = new System.Windows.Forms.Label();
-         this.myPrimitivesPage = new System.Windows.Forms.TabPage();
-         this.myTreeView = new System.Windows.Forms.TreeView();
-         this.myConsoleTabPage = new System.Windows.Forms.TabPage();
-         this.myConsoleRichTextBox = new System.Windows.Forms.RichTextBox();
-         this.myTimer = new System.Windows.Forms.Timer(this.components);
-         this.mySplitter = new System.Windows.Forms.Splitter();
-         this.myStatusStrip = new System.Windows.Forms.StatusStrip();
-         this.myToolStripStatusLabelTime = new System.Windows.Forms.ToolStripStatusLabel();
-         this.myToolStripStatusLabelRows = new System.Windows.Forms.ToolStripStatusLabel();
-         this.myMenuStrip = new System.Windows.Forms.MenuStrip();
-         this.myToolStripMenuItemFile = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemOpen = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemSave = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemSaveAs = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemExit = new System.Windows.Forms.ToolStripMenuItem();
-         this.testGridToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemQuery = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemExecute = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemExecSel = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemFreeze = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemStep = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemTools = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemLoadAss = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemTest = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemToolsSep = new System.Windows.Forms.ToolStripSeparator();
-         this.myToolStripMenuItemClearConsole = new System.Windows.Forms.ToolStripMenuItem();
-         this.myToolStripMenuItemSite = new System.Windows.Forms.ToolStripMenuItem();
-         this.libraryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
-         this.myTabControl.SuspendLayout();
-         this.myViewTabPage.SuspendLayout();
-         ((System.ComponentModel.ISupportInitialize)(this.myDataGrid)).BeginInit();
-         this.myCSharpTabPage.SuspendLayout();
-         this.myErrorTabPage.SuspendLayout();
-         this.myTableListTabPage.SuspendLayout();
-         this.myTableListTopPanel.SuspendLayout();
-         this.myPrimitivesPage.SuspendLayout();
-         this.myConsoleTabPage.SuspendLayout();
-         this.myStatusStrip.SuspendLayout();
-         this.myMenuStrip.SuspendLayout();
-         this.SuspendLayout();
+         myQueryTextBox = new System.Windows.Forms.RichTextBox();
+         myTabControl = new System.Windows.Forms.TabControl();
+         myViewTabPage = new System.Windows.Forms.TabPage();
+         myDataGrid = new HScrollGrid();
+         myCSharpTabPage = new System.Windows.Forms.TabPage();
+         myCSharpText = new System.Windows.Forms.RichTextBox();
+         myErrorTabPage = new System.Windows.Forms.TabPage();
+         myErrorTextBox = new System.Windows.Forms.RichTextBox();
+         myTableListTabPage = new System.Windows.Forms.TabPage();
+         myTableListBox = new Utilities.XPControls.ListBoxXP();
+         myTableListTopPanel = new System.Windows.Forms.Panel();
+         myTableFilterBox = new System.Windows.Forms.TextBox();
+         myTableFilterLabel = new System.Windows.Forms.Label();
+         myPrimitivesPage = new System.Windows.Forms.TabPage();
+         myTreeView = new System.Windows.Forms.TreeView();
+         myConsoleTabPage = new System.Windows.Forms.TabPage();
+         myConsoleRichTextBox = new System.Windows.Forms.RichTextBox();
+         myTimer = new System.Windows.Forms.Timer(components);
+         mySplitter = new System.Windows.Forms.Splitter();
+         myStatusStrip = new System.Windows.Forms.StatusStrip();
+         myToolStripStatusLabelTime = new System.Windows.Forms.ToolStripStatusLabel();
+         myToolStripStatusLabelRows = new System.Windows.Forms.ToolStripStatusLabel();
+         toolStripStatusLabel1 = new System.Windows.Forms.ToolStripStatusLabel();
+         myMenuStrip = new System.Windows.Forms.MenuStrip();
+         myToolStripMenuItemFile = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemOpen = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemSave = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemSaveAs = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemExit = new System.Windows.Forms.ToolStripMenuItem();
+         testGridToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemQuery = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemExecute = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemExecSel = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemFreeze = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemStep = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemTools = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemLoadAss = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemTest = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemToolsSep = new System.Windows.Forms.ToolStripSeparator();
+         myToolStripMenuItemClearConsole = new System.Windows.Forms.ToolStripMenuItem();
+         myToolStripMenuItemSite = new System.Windows.Forms.ToolStripMenuItem();
+         libraryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+         myTabControl.SuspendLayout();
+         myViewTabPage.SuspendLayout();
+         ((System.ComponentModel.ISupportInitialize)myDataGrid).BeginInit();
+         myCSharpTabPage.SuspendLayout();
+         myErrorTabPage.SuspendLayout();
+         myTableListTabPage.SuspendLayout();
+         myTableListTopPanel.SuspendLayout();
+         myPrimitivesPage.SuspendLayout();
+         myConsoleTabPage.SuspendLayout();
+         myStatusStrip.SuspendLayout();
+         myMenuStrip.SuspendLayout();
+         SuspendLayout();
          // 
          // myQueryTextBox
          // 
-         this.myQueryTextBox.AutoWordSelection = true;
-         this.myQueryTextBox.BackColor = System.Drawing.Color.White;
-         this.myQueryTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myQueryTextBox.Dock = System.Windows.Forms.DockStyle.Top;
-         this.myQueryTextBox.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         this.myQueryTextBox.ForeColor = System.Drawing.Color.Black;
-         this.myQueryTextBox.Location = new System.Drawing.Point(0, 24);
-         this.myQueryTextBox.Name = "myQueryTextBox";
-         this.myQueryTextBox.Size = new System.Drawing.Size(904, 208);
-         this.myQueryTextBox.TabIndex = 0;
-         this.myQueryTextBox.Text = "";
+         myQueryTextBox.AutoWordSelection = true;
+         myQueryTextBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myQueryTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myQueryTextBox.Dock = System.Windows.Forms.DockStyle.Top;
+         myQueryTextBox.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         myQueryTextBox.ForeColor = System.Drawing.Color.White;
+         myQueryTextBox.Location = new System.Drawing.Point(0, 24);
+         myQueryTextBox.Name = "myQueryTextBox";
+         myQueryTextBox.Size = new System.Drawing.Size(904, 208);
+         myQueryTextBox.TabIndex = 0;
+         myQueryTextBox.Text = "";
          // 
          // myTabControl
          // 
-         this.myTabControl.Appearance = System.Windows.Forms.TabAppearance.FlatButtons;
-         this.myTabControl.Controls.Add(this.myViewTabPage);
-         this.myTabControl.Controls.Add(this.myCSharpTabPage);
-         this.myTabControl.Controls.Add(this.myErrorTabPage);
-         this.myTabControl.Controls.Add(this.myTableListTabPage);
-         this.myTabControl.Controls.Add(this.myPrimitivesPage);
-         this.myTabControl.Controls.Add(this.myConsoleTabPage);
-         this.myTabControl.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myTabControl.Location = new System.Drawing.Point(0, 232);
-         this.myTabControl.Multiline = true;
-         this.myTabControl.Name = "myTabControl";
-         this.myTabControl.SelectedIndex = 0;
-         this.myTabControl.Size = new System.Drawing.Size(904, 335);
-         this.myTabControl.TabIndex = 1;
+         myTabControl.Appearance = System.Windows.Forms.TabAppearance.FlatButtons;
+         myTabControl.Controls.Add(myViewTabPage);
+         myTabControl.Controls.Add(myCSharpTabPage);
+         myTabControl.Controls.Add(myErrorTabPage);
+         myTabControl.Controls.Add(myTableListTabPage);
+         myTabControl.Controls.Add(myPrimitivesPage);
+         myTabControl.Controls.Add(myConsoleTabPage);
+         myTabControl.Dock = System.Windows.Forms.DockStyle.Fill;
+         myTabControl.Location = new System.Drawing.Point(0, 232);
+         myTabControl.Multiline = true;
+         myTabControl.Name = "myTabControl";
+         myTabControl.SelectedIndex = 0;
+         myTabControl.Size = new System.Drawing.Size(904, 335);
+         myTabControl.TabIndex = 1;
          // 
          // myViewTabPage
          // 
-         this.myViewTabPage.Controls.Add(this.myDataGrid);
-         this.myViewTabPage.Location = new System.Drawing.Point(4, 26);
-         this.myViewTabPage.Name = "myViewTabPage";
-         this.myViewTabPage.Size = new System.Drawing.Size(896, 305);
-         this.myViewTabPage.TabIndex = 0;
-         this.myViewTabPage.Text = "View";
+         myViewTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myViewTabPage.Controls.Add(myDataGrid);
+         myViewTabPage.ForeColor = System.Drawing.Color.White;
+         myViewTabPage.Location = new System.Drawing.Point(4, 26);
+         myViewTabPage.Name = "myViewTabPage";
+         myViewTabPage.Size = new System.Drawing.Size(896, 305);
+         myViewTabPage.TabIndex = 0;
+         myViewTabPage.Text = "View";
          // 
          // myDataGrid
          // 
-         this.myDataGrid.BackgroundColor = System.Drawing.Color.Gainsboro;
-         this.myDataGrid.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
-         // this.myDataGrid.CaptionBackColor = System.Drawing.Color.Wheat;
-         // this.myDataGrid.CaptionFont = new System.Drawing.Font("Comic Sans MS", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         // this.myDataGrid.CaptionForeColor = System.Drawing.Color.Black;
-         this.myDataGrid.DataMember = "";
-         this.myDataGrid.Dock = System.Windows.Forms.DockStyle.Fill;
-         // this.myDataGrid.FlatMode = true;
-         // this.myDataGrid.GridLineColor = System.Drawing.Color.Gainsboro;
-         // this.myDataGrid.HeaderBackColor = System.Drawing.Color.Gainsboro;
-         // this.myDataGrid.HeaderFont = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         // this.myDataGrid.HeaderForeColor = System.Drawing.Color.Black;
-         this.myDataGrid.HorizScrollBarValue = 0;
-         this.myDataGrid.Location = new System.Drawing.Point(0, 0);
-         this.myDataGrid.Name = "myDataGrid";
-         // this.myDataGrid.PreferredRowHeight = 20;
-         this.myDataGrid.ReadOnly = true;
-         this.myDataGrid.RowHeadersVisible = false;
-         // this.myDataGrid.RowHeaderWidth = 0;
-         this.myDataGrid.Size = new System.Drawing.Size(896, 305);
-         this.myDataGrid.TabIndex = 0;
-         this.myDataGrid.MouseEnter += new System.EventHandler(this.myDataGrid_MouseEnter);
-         this.myDataGrid.MouseLeave += new System.EventHandler(this.myDataGrid_MouseLeave);
+         myDataGrid.BackgroundColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myDataGrid.ColumnHeadersBorderStyle = System.Windows.Forms.DataGridViewHeaderBorderStyle.Single;
+         myDataGrid.ColumnHeadersHeight = 32;
+         dataGridViewCellStyle1.Alignment = System.Windows.Forms.DataGridViewContentAlignment.MiddleLeft;
+         dataGridViewCellStyle1.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         dataGridViewCellStyle1.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         dataGridViewCellStyle1.ForeColor = System.Drawing.Color.White;
+         dataGridViewCellStyle1.SelectionBackColor = System.Drawing.Color.Green;
+         dataGridViewCellStyle1.SelectionForeColor = System.Drawing.Color.Yellow;
+         dataGridViewCellStyle1.WrapMode = System.Windows.Forms.DataGridViewTriState.False;
+         myDataGrid.DefaultCellStyle = dataGridViewCellStyle1;
+         myDataGrid.Dock = System.Windows.Forms.DockStyle.Fill;
+         myDataGrid.GridColor = System.Drawing.Color.DarkGray;
+         myDataGrid.HorizScrollBarValue = 0;
+         myDataGrid.Location = new System.Drawing.Point(0, 0);
+         myDataGrid.Name = "myDataGrid";
+         myDataGrid.ReadOnly = true;
+         myDataGrid.RowHeadersBorderStyle = System.Windows.Forms.DataGridViewHeaderBorderStyle.None;
+         dataGridViewCellStyle2.Alignment = System.Windows.Forms.DataGridViewContentAlignment.MiddleLeft;
+         dataGridViewCellStyle2.BackColor = System.Drawing.SystemColors.Control;
+         dataGridViewCellStyle2.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         dataGridViewCellStyle2.ForeColor = System.Drawing.Color.White;
+         dataGridViewCellStyle2.SelectionBackColor = System.Drawing.Color.Yellow;
+         dataGridViewCellStyle2.SelectionForeColor = System.Drawing.Color.DimGray;
+         dataGridViewCellStyle2.WrapMode = System.Windows.Forms.DataGridViewTriState.True;
+         myDataGrid.RowHeadersDefaultCellStyle = dataGridViewCellStyle2;
+         myDataGrid.RowHeadersVisible = false;
+         myDataGrid.RowHeadersWidth = 82;
+         myDataGrid.Size = new System.Drawing.Size(896, 305);
+         myDataGrid.TabIndex = 0;
+         myDataGrid.MouseEnter += myDataGrid_MouseEnter;
+         myDataGrid.MouseLeave += myDataGrid_MouseLeave;
          // 
          // myCSharpTabPage
          // 
-         this.myCSharpTabPage.Controls.Add(this.myCSharpText);
-         this.myCSharpTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myCSharpTabPage.Name = "myCSharpTabPage";
-         this.myCSharpTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myCSharpTabPage.TabIndex = 2;
-         this.myCSharpTabPage.Text = "C# Class";
+         myCSharpTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myCSharpTabPage.Controls.Add(myCSharpText);
+         myCSharpTabPage.ForeColor = System.Drawing.Color.White;
+         myCSharpTabPage.Location = new System.Drawing.Point(4, 27);
+         myCSharpTabPage.Name = "myCSharpTabPage";
+         myCSharpTabPage.Size = new System.Drawing.Size(896, 304);
+         myCSharpTabPage.TabIndex = 2;
+         myCSharpTabPage.Text = "C# Class";
          // 
          // myCSharpText
          // 
-         this.myCSharpText.BackColor = System.Drawing.Color.White;
-         this.myCSharpText.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myCSharpText.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myCSharpText.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         this.myCSharpText.Location = new System.Drawing.Point(0, 0);
-         this.myCSharpText.Name = "myCSharpText";
-         this.myCSharpText.ReadOnly = true;
-         this.myCSharpText.Size = new System.Drawing.Size(896, 306);
-         this.myCSharpText.TabIndex = 0;
-         this.myCSharpText.Text = "";
-         this.myCSharpText.MouseEnter += new System.EventHandler(this.myCSharpText_MouseEnter);
-         this.myCSharpText.MouseLeave += new System.EventHandler(this.myCSharpText_MouseLeave);
+         myCSharpText.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myCSharpText.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myCSharpText.Dock = System.Windows.Forms.DockStyle.Fill;
+         myCSharpText.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         myCSharpText.ForeColor = System.Drawing.Color.White;
+         myCSharpText.Location = new System.Drawing.Point(0, 0);
+         myCSharpText.Name = "myCSharpText";
+         myCSharpText.ReadOnly = true;
+         myCSharpText.Size = new System.Drawing.Size(896, 304);
+         myCSharpText.TabIndex = 0;
+         myCSharpText.Text = "";
+         myCSharpText.MouseEnter += myCSharpText_MouseEnter;
+         myCSharpText.MouseLeave += myCSharpText_MouseLeave;
          // 
          // myErrorTabPage
          // 
-         this.myErrorTabPage.Controls.Add(this.myErrorTextBox);
-         this.myErrorTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myErrorTabPage.Name = "myErrorTabPage";
-         this.myErrorTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myErrorTabPage.TabIndex = 1;
-         this.myErrorTabPage.Text = "Error";
+         myErrorTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myErrorTabPage.Controls.Add(myErrorTextBox);
+         myErrorTabPage.ForeColor = System.Drawing.Color.White;
+         myErrorTabPage.Location = new System.Drawing.Point(4, 27);
+         myErrorTabPage.Name = "myErrorTabPage";
+         myErrorTabPage.Size = new System.Drawing.Size(896, 304);
+         myErrorTabPage.TabIndex = 1;
+         myErrorTabPage.Text = "Error";
          // 
          // myErrorTextBox
          // 
-         this.myErrorTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myErrorTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myErrorTextBox.ForeColor = System.Drawing.Color.Red;
-         this.myErrorTextBox.Location = new System.Drawing.Point(0, 0);
-         this.myErrorTextBox.Name = "myErrorTextBox";
-         this.myErrorTextBox.ReadOnly = true;
-         this.myErrorTextBox.Size = new System.Drawing.Size(896, 306);
-         this.myErrorTextBox.TabIndex = 1;
-         this.myErrorTextBox.Text = "";
+         myErrorTextBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myErrorTextBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myErrorTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
+         myErrorTextBox.ForeColor = System.Drawing.Color.Red;
+         myErrorTextBox.Location = new System.Drawing.Point(0, 0);
+         myErrorTextBox.Name = "myErrorTextBox";
+         myErrorTextBox.ReadOnly = true;
+         myErrorTextBox.Size = new System.Drawing.Size(896, 304);
+         myErrorTextBox.TabIndex = 1;
+         myErrorTextBox.Text = "";
          // 
          // myTableListTabPage
          // 
-         this.myTableListTabPage.Controls.Add(this.myTableListBox);
-         this.myTableListTabPage.Controls.Add(this.myTableListTopPanel);
-         this.myTableListTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myTableListTabPage.Name = "myTableListTabPage";
-         this.myTableListTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myTableListTabPage.TabIndex = 3;
-         this.myTableListTabPage.Text = "Table list";
+         myTableListTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableListTabPage.Controls.Add(myTableListBox);
+         myTableListTabPage.Controls.Add(myTableListTopPanel);
+         myTableListTabPage.ForeColor = System.Drawing.Color.White;
+         myTableListTabPage.Location = new System.Drawing.Point(4, 27);
+         myTableListTabPage.Name = "myTableListTabPage";
+         myTableListTabPage.Size = new System.Drawing.Size(896, 304);
+         myTableListTabPage.TabIndex = 3;
+         myTableListTabPage.Text = "Table list";
          // 
          // myTableListBox
          // 
-         this.myTableListBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myTableListBox.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myTableListBox.DrawMode = System.Windows.Forms.DrawMode.OwnerDrawFixed;
-         this.myTableListBox.ItemHeight = 14;
-         this.myTableListBox.Location = new System.Drawing.Point(0, 31);
-         this.myTableListBox.Name = "myTableListBox";
-         this.myTableListBox.Size = new System.Drawing.Size(896, 275);
-         this.myTableListBox.TabIndex = 1;
-         this.myTableListBox.DoubleClick += new System.EventHandler(this.myTableListBox_DoubleClick);
-         this.myTableListBox.MouseEnter += new System.EventHandler(this.myTableListBox_MouseEnter);
-         this.myTableListBox.MouseLeave += new System.EventHandler(this.myTableListBox_MouseLeave);
+         myTableListBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableListBox.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myTableListBox.Dock = System.Windows.Forms.DockStyle.Fill;
+         myTableListBox.DrawMode = System.Windows.Forms.DrawMode.OwnerDrawFixed;
+         myTableListBox.ForeColor = System.Drawing.Color.White;
+         myTableListBox.ItemHeight = 14;
+         myTableListBox.Location = new System.Drawing.Point(0, 31);
+         myTableListBox.Name = "myTableListBox";
+         myTableListBox.Size = new System.Drawing.Size(896, 273);
+         myTableListBox.TabIndex = 1;
+         myTableListBox.DoubleClick += myTableListBox_DoubleClick;
+         myTableListBox.MouseEnter += myTableListBox_MouseEnter;
+         myTableListBox.MouseLeave += myTableListBox_MouseLeave;
          // 
          // myTableListTopPanel
          // 
-         this.myTableListTopPanel.Controls.Add(this.myTableFilterBox);
-         this.myTableListTopPanel.Controls.Add(this.myTableFilterLabel);
-         this.myTableListTopPanel.Dock = System.Windows.Forms.DockStyle.Top;
-         this.myTableListTopPanel.Location = new System.Drawing.Point(0, 0);
-         this.myTableListTopPanel.Name = "myTableListTopPanel";
-         this.myTableListTopPanel.Size = new System.Drawing.Size(896, 31);
-         this.myTableListTopPanel.TabIndex = 4;
+         myTableListTopPanel.Controls.Add(myTableFilterBox);
+         myTableListTopPanel.Controls.Add(myTableFilterLabel);
+         myTableListTopPanel.Dock = System.Windows.Forms.DockStyle.Top;
+         myTableListTopPanel.Location = new System.Drawing.Point(0, 0);
+         myTableListTopPanel.Name = "myTableListTopPanel";
+         myTableListTopPanel.Size = new System.Drawing.Size(896, 31);
+         myTableListTopPanel.TabIndex = 4;
          // 
          // myTableFilterBox
          // 
-         this.myTableFilterBox.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
-            | System.Windows.Forms.AnchorStyles.Left) 
-            | System.Windows.Forms.AnchorStyles.Right)));
-         this.myTableFilterBox.Location = new System.Drawing.Point(58, 3);
-         this.myTableFilterBox.Name = "myTableFilterBox";
-         this.myTableFilterBox.Size = new System.Drawing.Size(835, 20);
-         this.myTableFilterBox.TabIndex = 2;
-         this.myTableFilterBox.TextChanged += new System.EventHandler(this.myTableFilterBox_TextChanged);
+         myTableFilterBox.Anchor = System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right;
+         myTableFilterBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableFilterBox.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
+         myTableFilterBox.ForeColor = System.Drawing.Color.White;
+         myTableFilterBox.Location = new System.Drawing.Point(58, 3);
+         myTableFilterBox.Name = "myTableFilterBox";
+         myTableFilterBox.Size = new System.Drawing.Size(835, 20);
+         myTableFilterBox.TabIndex = 2;
+         myTableFilterBox.TextChanged += myTableFilterBox_TextChanged;
          // 
          // myTableFilterLabel
          // 
-         this.myTableFilterLabel.AutoSize = true;
-         this.myTableFilterLabel.Location = new System.Drawing.Point(3, 3);
-         this.myTableFilterLabel.Name = "myTableFilterLabel";
-         this.myTableFilterLabel.Size = new System.Drawing.Size(49, 14);
-         this.myTableFilterLabel.TabIndex = 3;
-         this.myTableFilterLabel.Text = "Filter";
+         myTableFilterLabel.AutoSize = true;
+         myTableFilterLabel.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTableFilterLabel.ForeColor = System.Drawing.Color.White;
+         myTableFilterLabel.Location = new System.Drawing.Point(3, 3);
+         myTableFilterLabel.Name = "myTableFilterLabel";
+         myTableFilterLabel.Size = new System.Drawing.Size(49, 14);
+         myTableFilterLabel.TabIndex = 3;
+         myTableFilterLabel.Text = "Filter";
+         myTableFilterLabel.Click += myTableFilterLabel_Click;
          // 
          // myPrimitivesPage
          // 
-         this.myPrimitivesPage.Controls.Add(this.myTreeView);
-         this.myPrimitivesPage.Location = new System.Drawing.Point(4, 25);
-         this.myPrimitivesPage.Name = "myPrimitivesPage";
-         this.myPrimitivesPage.Size = new System.Drawing.Size(896, 306);
-         this.myPrimitivesPage.TabIndex = 4;
-         this.myPrimitivesPage.Text = "Language primitives";
+         myPrimitivesPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myPrimitivesPage.Controls.Add(myTreeView);
+         myPrimitivesPage.ForeColor = System.Drawing.Color.White;
+         myPrimitivesPage.Location = new System.Drawing.Point(4, 27);
+         myPrimitivesPage.Name = "myPrimitivesPage";
+         myPrimitivesPage.Size = new System.Drawing.Size(896, 304);
+         myPrimitivesPage.TabIndex = 4;
+         myPrimitivesPage.Text = "Language primitives";
          // 
          // myTreeView
          // 
-         this.myTreeView.BorderStyle = System.Windows.Forms.BorderStyle.None;
-         this.myTreeView.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myTreeView.Location = new System.Drawing.Point(0, 0);
-         this.myTreeView.Name = "myTreeView";
-         this.myTreeView.Size = new System.Drawing.Size(896, 306);
-         this.myTreeView.TabIndex = 0;
+         myTreeView.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myTreeView.BorderStyle = System.Windows.Forms.BorderStyle.None;
+         myTreeView.Dock = System.Windows.Forms.DockStyle.Fill;
+         myTreeView.ForeColor = System.Drawing.Color.White;
+         myTreeView.Location = new System.Drawing.Point(0, 0);
+         myTreeView.Name = "myTreeView";
+         myTreeView.Size = new System.Drawing.Size(896, 304);
+         myTreeView.TabIndex = 0;
          // 
          // myConsoleTabPage
          // 
-         this.myConsoleTabPage.Controls.Add(this.myConsoleRichTextBox);
-         this.myConsoleTabPage.Location = new System.Drawing.Point(4, 25);
-         this.myConsoleTabPage.Name = "myConsoleTabPage";
-         this.myConsoleTabPage.Padding = new System.Windows.Forms.Padding(3);
-         this.myConsoleTabPage.Size = new System.Drawing.Size(896, 306);
-         this.myConsoleTabPage.TabIndex = 5;
-         this.myConsoleTabPage.Text = "Console";
-         this.myConsoleTabPage.UseVisualStyleBackColor = true;
+         myConsoleTabPage.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myConsoleTabPage.Controls.Add(myConsoleRichTextBox);
+         myConsoleTabPage.ForeColor = System.Drawing.Color.White;
+         myConsoleTabPage.Location = new System.Drawing.Point(4, 27);
+         myConsoleTabPage.Name = "myConsoleTabPage";
+         myConsoleTabPage.Padding = new System.Windows.Forms.Padding(3);
+         myConsoleTabPage.Size = new System.Drawing.Size(896, 304);
+         myConsoleTabPage.TabIndex = 5;
+         myConsoleTabPage.Text = "Console";
          // 
          // myConsoleRichTextBox
          // 
-         this.myConsoleRichTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
-         this.myConsoleRichTextBox.Location = new System.Drawing.Point(3, 3);
-         this.myConsoleRichTextBox.Name = "myConsoleRichTextBox";
-         this.myConsoleRichTextBox.Size = new System.Drawing.Size(890, 300);
-         this.myConsoleRichTextBox.TabIndex = 0;
-         this.myConsoleRichTextBox.Text = "";
+         myConsoleRichTextBox.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myConsoleRichTextBox.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
+         myConsoleRichTextBox.Dock = System.Windows.Forms.DockStyle.Fill;
+         myConsoleRichTextBox.ForeColor = System.Drawing.Color.White;
+         myConsoleRichTextBox.Location = new System.Drawing.Point(3, 3);
+         myConsoleRichTextBox.Name = "myConsoleRichTextBox";
+         myConsoleRichTextBox.Size = new System.Drawing.Size(890, 298);
+         myConsoleRichTextBox.TabIndex = 0;
+         myConsoleRichTextBox.Text = "";
          // 
          // myTimer
          // 
-         this.myTimer.Enabled = true;
-         this.myTimer.Interval = 150;
-         this.myTimer.Tick += new System.EventHandler(this.myTimer_Tick);
+         myTimer.Enabled = true;
+         myTimer.Interval = 150;
+         myTimer.Tick += myTimer_Tick;
          // 
          // mySplitter
          // 
-         this.mySplitter.Dock = System.Windows.Forms.DockStyle.Top;
-         this.mySplitter.Location = new System.Drawing.Point(0, 232);
-         this.mySplitter.Name = "mySplitter";
-         this.mySplitter.Size = new System.Drawing.Size(904, 3);
-         this.mySplitter.TabIndex = 3;
-         this.mySplitter.TabStop = false;
+         mySplitter.Dock = System.Windows.Forms.DockStyle.Top;
+         mySplitter.Location = new System.Drawing.Point(0, 232);
+         mySplitter.Name = "mySplitter";
+         mySplitter.Size = new System.Drawing.Size(904, 4);
+         mySplitter.TabIndex = 3;
+         mySplitter.TabStop = false;
          // 
          // myStatusStrip
          // 
-         this.myStatusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripStatusLabelTime,
-            this.myToolStripStatusLabelRows});
-         this.myStatusStrip.Location = new System.Drawing.Point(0, 567);
-         this.myStatusStrip.Name = "myStatusStrip";
-         this.myStatusStrip.Size = new System.Drawing.Size(904, 22);
-         this.myStatusStrip.TabIndex = 4;
+         myStatusStrip.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myStatusStrip.ImageScalingSize = new System.Drawing.Size(32, 32);
+         myStatusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripStatusLabelTime, myToolStripStatusLabelRows, toolStripStatusLabel1 });
+         myStatusStrip.Location = new System.Drawing.Point(0, 567);
+         myStatusStrip.Name = "myStatusStrip";
+         myStatusStrip.Size = new System.Drawing.Size(904, 22);
+         myStatusStrip.TabIndex = 4;
          // 
          // myToolStripStatusLabelTime
          // 
-         this.myToolStripStatusLabelTime.Name = "myToolStripStatusLabelTime";
-         this.myToolStripStatusLabelTime.Padding = new System.Windows.Forms.Padding(0, 0, 8, 0);
-         this.myToolStripStatusLabelTime.Size = new System.Drawing.Size(8, 17);
+         myToolStripStatusLabelTime.Name = "myToolStripStatusLabelTime";
+         myToolStripStatusLabelTime.Padding = new System.Windows.Forms.Padding(0, 0, 8, 0);
+         myToolStripStatusLabelTime.Size = new System.Drawing.Size(8, 17);
          // 
          // myToolStripStatusLabelRows
          // 
-         this.myToolStripStatusLabelRows.Name = "myToolStripStatusLabelRows";
-         this.myToolStripStatusLabelRows.Size = new System.Drawing.Size(0, 17);
+         myToolStripStatusLabelRows.Name = "myToolStripStatusLabelRows";
+         myToolStripStatusLabelRows.Size = new System.Drawing.Size(0, 17);
+         // 
+         // toolStripStatusLabel1
+         // 
+         toolStripStatusLabel1.ForeColor = System.Drawing.Color.White;
+         toolStripStatusLabel1.Name = "toolStripStatusLabel1";
+         toolStripStatusLabel1.Size = new System.Drawing.Size(0, 17);
          // 
          // myMenuStrip
          // 
-         this.myMenuStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemFile,
-            this.myToolStripMenuItemQuery,
-            this.myToolStripMenuItemTools,
-            this.myToolStripMenuItemSite,
-            this.libraryToolStripMenuItem});
-         this.myMenuStrip.Location = new System.Drawing.Point(0, 0);
-         this.myMenuStrip.Name = "myMenuStrip";
-         this.myMenuStrip.Size = new System.Drawing.Size(904, 24);
-         this.myMenuStrip.TabIndex = 5;
-         this.myMenuStrip.Text = "menuStrip1";
+         myMenuStrip.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myMenuStrip.ImageScalingSize = new System.Drawing.Size(32, 32);
+         myMenuStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemFile, myToolStripMenuItemQuery, myToolStripMenuItemTools, myToolStripMenuItemSite, libraryToolStripMenuItem });
+         myMenuStrip.Location = new System.Drawing.Point(0, 0);
+         myMenuStrip.Name = "myMenuStrip";
+         myMenuStrip.Size = new System.Drawing.Size(904, 24);
+         myMenuStrip.TabIndex = 5;
+         myMenuStrip.Text = "menuStrip1";
          // 
          // myToolStripMenuItemFile
          // 
-         this.myToolStripMenuItemFile.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemOpen,
-            this.myToolStripMenuItemSave,
-            this.myToolStripMenuItemSaveAs,
-            this.myToolStripMenuItemExit,
-            this.testGridToolStripMenuItem});
-         this.myToolStripMenuItemFile.Name = "myToolStripMenuItemFile";
-         this.myToolStripMenuItemFile.Size = new System.Drawing.Size(37, 20);
-         this.myToolStripMenuItemFile.Text = "&File";
+         myToolStripMenuItemFile.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemOpen, myToolStripMenuItemSave, myToolStripMenuItemSaveAs, myToolStripMenuItemExit, testGridToolStripMenuItem });
+         myToolStripMenuItemFile.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemFile.Name = "myToolStripMenuItemFile";
+         myToolStripMenuItemFile.Size = new System.Drawing.Size(37, 20);
+         myToolStripMenuItemFile.Text = "&File";
          // 
          // myToolStripMenuItemOpen
          // 
-         this.myToolStripMenuItemOpen.Image = global::DataStreamQueryAnalyzer.Properties.Resources.File_Open_icon16;
-         this.myToolStripMenuItemOpen.Name = "myToolStripMenuItemOpen";
-         this.myToolStripMenuItemOpen.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemOpen.Text = "&Open...";
-         this.myToolStripMenuItemOpen.Click += new System.EventHandler(this.myMenuItemOpenClick);
+         myToolStripMenuItemOpen.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemOpen.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemOpen.Image = DataStreamQueryAnalyzer.Properties.Resources.File_Open_icon16;
+         myToolStripMenuItemOpen.Name = "myToolStripMenuItemOpen";
+         myToolStripMenuItemOpen.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemOpen.Text = "&Open...";
+         myToolStripMenuItemOpen.Click += myMenuItemOpenClick;
          // 
          // myToolStripMenuItemSave
          // 
-         this.myToolStripMenuItemSave.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Save_icon16;
-         this.myToolStripMenuItemSave.Name = "myToolStripMenuItemSave";
-         this.myToolStripMenuItemSave.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemSave.Text = "Save...";
-         this.myToolStripMenuItemSave.Click += new System.EventHandler(this.myMenuItemSaveClick);
+         myToolStripMenuItemSave.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemSave.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemSave.Image = DataStreamQueryAnalyzer.Properties.Resources.Save_icon16;
+         myToolStripMenuItemSave.Name = "myToolStripMenuItemSave";
+         myToolStripMenuItemSave.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemSave.Text = "Save...";
+         myToolStripMenuItemSave.Click += myMenuItemSaveClick;
          // 
          // myToolStripMenuItemSaveAs
          // 
-         this.myToolStripMenuItemSaveAs.Name = "myToolStripMenuItemSaveAs";
-         this.myToolStripMenuItemSaveAs.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemSaveAs.Text = "Save As...";
-         this.myToolStripMenuItemSaveAs.Click += new System.EventHandler(this.myMenuItemSaveAsClick);
+         myToolStripMenuItemSaveAs.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemSaveAs.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemSaveAs.Name = "myToolStripMenuItemSaveAs";
+         myToolStripMenuItemSaveAs.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemSaveAs.Text = "Save As...";
+         myToolStripMenuItemSaveAs.Click += myMenuItemSaveAsClick;
          // 
          // myToolStripMenuItemExit
          // 
-         this.myToolStripMenuItemExit.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Turn_Off_icon16;
-         this.myToolStripMenuItemExit.Name = "myToolStripMenuItemExit";
-         this.myToolStripMenuItemExit.Size = new System.Drawing.Size(123, 22);
-         this.myToolStripMenuItemExit.Text = "Exit";
-         this.myToolStripMenuItemExit.Click += new System.EventHandler(this.myToolStripMenuItemExit_Click);
+         myToolStripMenuItemExit.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemExit.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemExit.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Turn_Off_icon16;
+         myToolStripMenuItemExit.Name = "myToolStripMenuItemExit";
+         myToolStripMenuItemExit.Size = new System.Drawing.Size(139, 38);
+         myToolStripMenuItemExit.Text = "Exit";
+         myToolStripMenuItemExit.Click += myToolStripMenuItemExit_Click;
          // 
          // testGridToolStripMenuItem
          // 
-         this.testGridToolStripMenuItem.Name = "testGridToolStripMenuItem";
-         this.testGridToolStripMenuItem.Size = new System.Drawing.Size(123, 22);
-         this.testGridToolStripMenuItem.Text = "Test Grid";
-         this.testGridToolStripMenuItem.Click += new System.EventHandler(this.testGridToolStripMenuItem_Click);
+         testGridToolStripMenuItem.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         testGridToolStripMenuItem.ForeColor = System.Drawing.Color.White;
+         testGridToolStripMenuItem.Name = "testGridToolStripMenuItem";
+         testGridToolStripMenuItem.Size = new System.Drawing.Size(139, 38);
+         testGridToolStripMenuItem.Text = "Test Grid";
+         testGridToolStripMenuItem.Click += testGridToolStripMenuItem_Click;
          // 
          // myToolStripMenuItemQuery
          // 
-         this.myToolStripMenuItemQuery.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemExecute,
-            this.myToolStripMenuItemExecSel,
-            this.myToolStripMenuItemFreeze,
-            this.myToolStripMenuItemStep});
-         this.myToolStripMenuItemQuery.Name = "myToolStripMenuItemQuery";
-         this.myToolStripMenuItemQuery.Size = new System.Drawing.Size(51, 20);
-         this.myToolStripMenuItemQuery.Text = "&Query";
+         myToolStripMenuItemQuery.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemExecute, myToolStripMenuItemExecSel, myToolStripMenuItemFreeze, myToolStripMenuItemStep });
+         myToolStripMenuItemQuery.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemQuery.Name = "myToolStripMenuItemQuery";
+         myToolStripMenuItemQuery.Size = new System.Drawing.Size(51, 20);
+         myToolStripMenuItemQuery.Text = "&Query";
          // 
          // myToolStripMenuItemExecute
          // 
-         this.myToolStripMenuItemExecute.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Play_icon16;
-         this.myToolStripMenuItemExecute.Name = "myToolStripMenuItemExecute";
-         this.myToolStripMenuItemExecute.ShortcutKeys = System.Windows.Forms.Keys.F11;
-         this.myToolStripMenuItemExecute.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemExecute.Text = "Execute";
-         this.myToolStripMenuItemExecute.Click += new System.EventHandler(this.myMenuItemExecuteQueryClick);
+         myToolStripMenuItemExecute.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemExecute.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemExecute.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Play_icon16;
+         myToolStripMenuItemExecute.Name = "myToolStripMenuItemExecute";
+         myToolStripMenuItemExecute.ShortcutKeys = System.Windows.Forms.Keys.F11;
+         myToolStripMenuItemExecute.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemExecute.Text = "Execute";
+         myToolStripMenuItemExecute.Click += myMenuItemExecuteQueryClick;
          // 
          // myToolStripMenuItemExecSel
          // 
-         this.myToolStripMenuItemExecSel.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Blank_Blue_icon16;
-         this.myToolStripMenuItemExecSel.Name = "myToolStripMenuItemExecSel";
-         this.myToolStripMenuItemExecSel.ShortcutKeys = System.Windows.Forms.Keys.F12;
-         this.myToolStripMenuItemExecSel.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemExecSel.Text = "Execute Selection";
-         this.myToolStripMenuItemExecSel.Click += new System.EventHandler(this.myToolStripMenuItemExecSel_Click);
+         myToolStripMenuItemExecSel.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemExecSel.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemExecSel.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Blank_Blue_icon16;
+         myToolStripMenuItemExecSel.Name = "myToolStripMenuItemExecSel";
+         myToolStripMenuItemExecSel.ShortcutKeys = System.Windows.Forms.Keys.F12;
+         myToolStripMenuItemExecSel.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemExecSel.Text = "Execute Selection";
+         myToolStripMenuItemExecSel.Click += myToolStripMenuItemExecSel_Click;
          // 
          // myToolStripMenuItemFreeze
          // 
-         this.myToolStripMenuItemFreeze.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Pause_icon16;
-         this.myToolStripMenuItemFreeze.Name = "myToolStripMenuItemFreeze";
-         this.myToolStripMenuItemFreeze.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemFreeze.Text = "Freeze";
-         this.myToolStripMenuItemFreeze.Click += new System.EventHandler(this.myToolStripMenuItemFreeze_Click);
+         myToolStripMenuItemFreeze.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemFreeze.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemFreeze.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Pause_icon16;
+         myToolStripMenuItemFreeze.Name = "myToolStripMenuItemFreeze";
+         myToolStripMenuItemFreeze.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemFreeze.Text = "Freeze";
+         myToolStripMenuItemFreeze.Click += myToolStripMenuItemFreeze_Click;
          // 
          // myToolStripMenuItemStep
          // 
-         this.myToolStripMenuItemStep.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Forward_icon16;
-         this.myToolStripMenuItemStep.Name = "myToolStripMenuItemStep";
-         this.myToolStripMenuItemStep.Size = new System.Drawing.Size(190, 22);
-         this.myToolStripMenuItemStep.Text = "Step";
-         this.myToolStripMenuItemStep.Click += new System.EventHandler(this.myToolStripMenuItemStep_Click);
+         myToolStripMenuItemStep.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemStep.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemStep.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Forward_icon16;
+         myToolStripMenuItemStep.Name = "myToolStripMenuItemStep";
+         myToolStripMenuItemStep.Size = new System.Drawing.Size(207, 38);
+         myToolStripMenuItemStep.Text = "Step";
+         myToolStripMenuItemStep.Click += myToolStripMenuItemStep_Click;
          // 
          // myToolStripMenuItemTools
          // 
-         this.myToolStripMenuItemTools.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
-            this.myToolStripMenuItemLoadAss,
-            this.myToolStripMenuItemTest,
-            this.myToolStripMenuItemToolsSep,
-            this.myToolStripMenuItemClearConsole});
-         this.myToolStripMenuItemTools.Name = "myToolStripMenuItemTools";
-         this.myToolStripMenuItemTools.Size = new System.Drawing.Size(48, 20);
-         this.myToolStripMenuItemTools.Text = "&Tools";
+         myToolStripMenuItemTools.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { myToolStripMenuItemLoadAss, myToolStripMenuItemTest, myToolStripMenuItemToolsSep, myToolStripMenuItemClearConsole });
+         myToolStripMenuItemTools.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemTools.Name = "myToolStripMenuItemTools";
+         myToolStripMenuItemTools.Size = new System.Drawing.Size(46, 20);
+         myToolStripMenuItemTools.Text = "&Tools";
          // 
          // myToolStripMenuItemLoadAss
          // 
-         this.myToolStripMenuItemLoadAss.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Upload_icon16;
-         this.myToolStripMenuItemLoadAss.Name = "myToolStripMenuItemLoadAss";
-         this.myToolStripMenuItemLoadAss.Size = new System.Drawing.Size(154, 22);
-         this.myToolStripMenuItemLoadAss.Text = "Load Assembly";
-         this.myToolStripMenuItemLoadAss.Click += new System.EventHandler(this.myToolStripMenuItemLoadAss_Click);
+         myToolStripMenuItemLoadAss.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Upload_icon16;
+         myToolStripMenuItemLoadAss.Name = "myToolStripMenuItemLoadAss";
+         myToolStripMenuItemLoadAss.Size = new System.Drawing.Size(170, 38);
+         myToolStripMenuItemLoadAss.Text = "Load Assembly";
+         myToolStripMenuItemLoadAss.Click += myToolStripMenuItemLoadAss_Click;
          // 
          // myToolStripMenuItemTest
          // 
-         this.myToolStripMenuItemTest.Image = global::DataStreamQueryAnalyzer.Properties.Resources.Button_Refresh_icon16;
-         this.myToolStripMenuItemTest.Name = "myToolStripMenuItemTest";
-         this.myToolStripMenuItemTest.ShortcutKeys = System.Windows.Forms.Keys.F4;
-         this.myToolStripMenuItemTest.Size = new System.Drawing.Size(154, 22);
-         this.myToolStripMenuItemTest.Text = "Test Resub";
-         this.myToolStripMenuItemTest.Click += new System.EventHandler(this.myToolStripMenuItemTest_Click);
+         myToolStripMenuItemTest.Image = DataStreamQueryAnalyzer.Properties.Resources.Button_Refresh_icon16;
+         myToolStripMenuItemTest.Name = "myToolStripMenuItemTest";
+         myToolStripMenuItemTest.ShortcutKeys = System.Windows.Forms.Keys.F4;
+         myToolStripMenuItemTest.Size = new System.Drawing.Size(170, 38);
+         myToolStripMenuItemTest.Text = "Test Resub";
+         myToolStripMenuItemTest.Click += myToolStripMenuItemTest_Click;
          // 
          // myToolStripMenuItemToolsSep
          // 
-         this.myToolStripMenuItemToolsSep.Name = "myToolStripMenuItemToolsSep";
-         this.myToolStripMenuItemToolsSep.Size = new System.Drawing.Size(151, 6);
+         myToolStripMenuItemToolsSep.Name = "myToolStripMenuItemToolsSep";
+         myToolStripMenuItemToolsSep.Size = new System.Drawing.Size(167, 6);
          // 
          // myToolStripMenuItemClearConsole
          // 
-         this.myToolStripMenuItemClearConsole.Name = "myToolStripMenuItemClearConsole";
-         this.myToolStripMenuItemClearConsole.Size = new System.Drawing.Size(154, 22);
-         this.myToolStripMenuItemClearConsole.Text = "Clear Console";
-         this.myToolStripMenuItemClearConsole.Click += new System.EventHandler(this.myToolStripMenuItemClearConsole_Click);
+         myToolStripMenuItemClearConsole.BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         myToolStripMenuItemClearConsole.ForeColor = System.Drawing.Color.White;
+         myToolStripMenuItemClearConsole.Name = "myToolStripMenuItemClearConsole";
+         myToolStripMenuItemClearConsole.Size = new System.Drawing.Size(170, 38);
+         myToolStripMenuItemClearConsole.Text = "Clear Console";
+         myToolStripMenuItemClearConsole.Click += myToolStripMenuItemClearConsole_Click;
          // 
          // myToolStripMenuItemSite
          // 
-         this.myToolStripMenuItemSite.Name = "myToolStripMenuItemSite";
-         this.myToolStripMenuItemSite.Size = new System.Drawing.Size(38, 20);
-         this.myToolStripMenuItemSite.Text = "&Site";
+         myToolStripMenuItemSite.Name = "myToolStripMenuItemSite";
+         myToolStripMenuItemSite.Size = new System.Drawing.Size(38, 20);
+         myToolStripMenuItemSite.Text = "&Site";
          // 
          // libraryToolStripMenuItem
          // 
-         this.libraryToolStripMenuItem.Name = "libraryToolStripMenuItem";
-         this.libraryToolStripMenuItem.Size = new System.Drawing.Size(55, 20);
-         this.libraryToolStripMenuItem.Text = "Library";
-         this.libraryToolStripMenuItem.Click += new System.EventHandler(this.libraryToolStripMenuItem_Click);
+         libraryToolStripMenuItem.Name = "libraryToolStripMenuItem";
+         libraryToolStripMenuItem.Size = new System.Drawing.Size(55, 20);
+         libraryToolStripMenuItem.Text = "Library";
+         libraryToolStripMenuItem.Click += libraryToolStripMenuItem_Click;
          // 
          // MainForm
          // 
-         this.AutoScaleBaseSize = new System.Drawing.Size(7, 13);
-         this.BackColor = System.Drawing.Color.Gainsboro;
-         this.ClientSize = new System.Drawing.Size(904, 589);
-         this.Controls.Add(this.mySplitter);
-         this.Controls.Add(this.myTabControl);
-         this.Controls.Add(this.myQueryTextBox);
-         this.Controls.Add(this.myMenuStrip);
-         this.Controls.Add(this.myStatusStrip);
-         this.Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-         this.ForeColor = System.Drawing.Color.Black;
-         this.Icon = ((System.Drawing.Icon)(resources.GetObject("$this.Icon")));
-         this.MainMenuStrip = this.myMenuStrip;
-         this.Name = "MainForm";
-         this.StartPosition = System.Windows.Forms.FormStartPosition.Manual;
-         this.Text = "DataStream Query Analyzer";
-         this.Closed += new System.EventHandler(this.MainForm_Closed);
-         this.myTabControl.ResumeLayout(false);
-         this.myViewTabPage.ResumeLayout(false);
-         ((System.ComponentModel.ISupportInitialize)(this.myDataGrid)).EndInit();
-         this.myCSharpTabPage.ResumeLayout(false);
-         this.myErrorTabPage.ResumeLayout(false);
-         this.myTableListTabPage.ResumeLayout(false);
-         this.myTableListTopPanel.ResumeLayout(false);
-         this.myTableListTopPanel.PerformLayout();
-         this.myPrimitivesPage.ResumeLayout(false);
-         this.myConsoleTabPage.ResumeLayout(false);
-         this.myStatusStrip.ResumeLayout(false);
-         this.myStatusStrip.PerformLayout();
-         this.myMenuStrip.ResumeLayout(false);
-         this.myMenuStrip.PerformLayout();
-         this.ResumeLayout(false);
-         this.PerformLayout();
-
+         AutoScaleBaseSize = new System.Drawing.Size(7, 13);
+         BackColor = System.Drawing.Color.FromArgb(36, 36, 49);
+         ClientSize = new System.Drawing.Size(904, 589);
+         Controls.Add(mySplitter);
+         Controls.Add(myTabControl);
+         Controls.Add(myQueryTextBox);
+         Controls.Add(myMenuStrip);
+         Controls.Add(myStatusStrip);
+         Font = new System.Drawing.Font("Courier New", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point);
+         ForeColor = System.Drawing.Color.White;
+         Icon = (System.Drawing.Icon)resources.GetObject("$this.Icon");
+         MainMenuStrip = myMenuStrip;
+         Name = "MainForm";
+         StartPosition = System.Windows.Forms.FormStartPosition.Manual;
+         Text = "BTS-cloud.io";
+         Closed += MainForm_Closed;
+         Load += MainForm_Load;
+         myTabControl.ResumeLayout(false);
+         myViewTabPage.ResumeLayout(false);
+         ((System.ComponentModel.ISupportInitialize)myDataGrid).EndInit();
+         myCSharpTabPage.ResumeLayout(false);
+         myErrorTabPage.ResumeLayout(false);
+         myTableListTabPage.ResumeLayout(false);
+         myTableListTopPanel.ResumeLayout(false);
+         myTableListTopPanel.PerformLayout();
+         myPrimitivesPage.ResumeLayout(false);
+         myConsoleTabPage.ResumeLayout(false);
+         myStatusStrip.ResumeLayout(false);
+         myStatusStrip.PerformLayout();
+         myMenuStrip.ResumeLayout(false);
+         myMenuStrip.PerformLayout();
+         ResumeLayout(false);
+         PerformLayout();
       }
       #endregion
 
@@ -561,5 +610,6 @@
       private System.Windows.Forms.Panel myTableListTopPanel;
       private System.Windows.Forms.ToolStripMenuItem testGridToolStripMenuItem;
       private System.Windows.Forms.ToolStripMenuItem libraryToolStripMenuItem;
+      private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel1;
    }
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -21,55 +21,65 @@ using Spark.DataStream;
 using Bts.DataStreamExpressions;
 // using BCG.Evaluator.DataStreamExpression;
 // using BCG.Utilities.Win32;
+using System.Threading.Tasks;
 
-namespace BCG
-{
-   public partial class MainForm : System.Windows.Forms.Form
-   {
+
+
+namespace BCG {
+   public partial class MainForm : System.Windows.Forms.Form {
+      // static async Task _run() {
+      //    var login = "bts";
+      //    var passwd = String.Empty;
+      //    Console.WriteLine("Starting DSQA... ");
+      //    // Log.notnull(login);
+      //    // Log.notnull(passwd);
+      //    var service_ctx = bts.api.core.ServiceContext.create();
+      //    await service_ctx.login(login, passwd);
+      // }
       private bool myChangeStyle = false;
       private PubSubRoute[] myRoutes = PubSub3Config.instance.getSubRouting();
       private PersistWindowState myWindowState;
       private string myLastFileName;
       private ViewStream myCurrentStream;
-      private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem> ();
+      private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
       // private string mySite = SitesCore.getMySiteName ();
       private string mySite = "TODO";
       private string myLastClassName = "MyView";
-      private ViewStream myAllTagsStream; 
-      private PubSubConfigureForm myPubSubConForm = new PubSubConfigureForm ();
+      private ViewStream myAllTagsStream;
+      private PubSubConfigureForm myPubSubConForm = new PubSubConfigureForm();
       private string myKQ;
       private TextWriter myConsoleOutput;
-      
-      public MainForm()
-      {        
+
+      public MainForm() {
+         //Task.Run(async () => await _run());
          InitializeComponent();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
          myKQ = "TODO";
          mySetText();
 
-         myWindowState = new PersistWindowState ();
+         myWindowState = new PersistWindowState();
          myWindowState.parent = this;
          myWindowState.saveState = false;
 
-         this.Load += new EventHandler (myLoad);
-         this.Closing += new CancelEventHandler (myClosing);         
+         this.Load += new EventHandler(myLoad);
+         this.Closing += new CancelEventHandler(myClosing);
+         myQueryTextBox.TextChanged += (o, e) => {
+            myQueryTextBox.ForeColor = Color.White;
+         };
       }
 
-      public string query_text
-      {
+      public string query_text {
          get { return myQueryTextBox.Text; }
          set { myQueryTextBox.Text = value; }
       }
-      private ViewStream myGetAllTagsStream ()
-      {
+      private ViewStream myGetAllTagsStream() {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
-         // return new FilterStream(new PublishedTagsStream(), String.Format("tag~='^{0}'", PubStreamPrefixer.prefix("")));
-         return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
+         return new FilterStream(new PubSubServerStateStream(), "tag~='DS_'");
+         //return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
       }
 
-      private void mySetText ()
-      {
-         this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty (myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
+      private void mySetText() {
+         this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
          myToolStripStatusLabelTime.Text = "Ready";
          myToolStripStatusLabelRows.Text = string.Empty;
       }
@@ -78,41 +88,36 @@ namespace BCG
       /// Gets the data in the form of a multilevel IDictionary
       /// </summary>
       /// <returns></returns>
-      private IDictionary myGetLibraryData ()
-      {
+      private IDictionary myGetLibraryData() {
          SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames ();
-         foreach (string pName in primitiveNames)
-         {
-            string aName = PrimitiveManager.instance.getAssemblyName (pName);
-            ICollection cNames = PrimitiveManager.instance.getConstructorSigns (pName);
-            if (!retVal.Contains (aName))
-               retVal.Add (aName, new SortedList ());
-            (retVal[aName] as IDictionary)[pName] = new ArrayList (cNames);
-            
+         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
+         foreach (string pName in primitiveNames) {
+            string aName = PrimitiveManager.instance.getAssemblyName(pName);
+            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
+            if (!retVal.Contains(aName))
+               retVal.Add(aName, new SortedList());
+            (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
+
          }
          foreach (DictionaryEntry outerDe in retVal)
             foreach (DictionaryEntry innerDe in (outerDe.Value as IDictionary))
-               (innerDe.Value as ArrayList).Sort ();
+               (innerDe.Value as ArrayList).Sort();
          return retVal;
       }
 
-      
-      private void myLoadAssemblies ()
-      {
-         myTreeView.Nodes.Clear ();
-         IDictionary libraryData = myGetLibraryData ();
-         foreach (DictionaryEntry de in libraryData)
-         {
-            TreeNode assemblyNode = new TreeNode (de.Key as String);
-            foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
-            { 
-               TreeNode primitiveNode = new TreeNode (innerDe.Key as string);
+
+      private void myLoadAssemblies() {
+         myTreeView.Nodes.Clear();
+         IDictionary libraryData = myGetLibraryData();
+         foreach (DictionaryEntry de in libraryData) {
+            TreeNode assemblyNode = new TreeNode(de.Key as String);
+            foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
+               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
                foreach (string con in (innerDe.Value as ICollection))
-                  primitiveNode.Nodes.Add (con);
-               assemblyNode.Nodes.Add (primitiveNode);
+                  primitiveNode.Nodes.Add(con);
+               assemblyNode.Nodes.Add(primitiveNode);
             }
-            myTreeView.Nodes.Add (assemblyNode);
+            myTreeView.Nodes.Add(assemblyNode);
          }
          /*
          myPrimitivesTextBox.Clear ();
@@ -126,27 +131,23 @@ namespace BCG
          }*/
       }
 
-      private void myHandleTableListChange (object o)
-      {
+      private void myHandleTableListChange(object o) {
          IBlueRowCollection rows = (o as DataPacket).newRows;
          myUpdateTableList();
       }
 
-      
+
       #region Persistence Keys
-      public int textBoxHeight
-      {
+      public int textBoxHeight {
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
       #endregion
-   
-      private void myTryExpression (string expStr)
-      {
-         myFreezeView (false); // force unfreeze the view
+
+      private void myTryExpression(string expStr) {
+         myFreezeView(false); // force unfreeze the view
          DataStreamExpression oldExp = myExp;
-         try 
-         {
+         try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
 
             if (exp == null)
@@ -156,7 +157,7 @@ namespace BCG
             myTabControl.SelectedIndex = 0;
             myErrorTextBox.Text = "";
             // myDataGrid.CaptionText = "";
-            
+
             if (myCurrentStream != null)
                myCurrentStream.onChange -= new BCG.DataStream.ChangeDelegate(myCallback);
             myCurrentStream = null;
@@ -164,7 +165,7 @@ namespace BCG
             myExp = exp;
             myCrono = new Crono();
             myCrono.start();
-            
+
             (myCurrentStream = myExp.getView(new Dictionary<string, StatementExpression>(0))).onChange += new DataStream.ChangeDelegate(myCallback);
             myChangeStyle = true;
             //new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback);
@@ -173,66 +174,55 @@ namespace BCG
                oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
                */
             //myQueryTextBox.Text = expStr;
-         }
-         catch (Exception exc)
-         {
+         } catch (Exception exc) {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
             myTabControl.SelectedIndex = 2;
-            myExp = oldExp;            
+            myExp = oldExp;
             myErrorTextBox.Text = exc.Message + "\n";
-            if(exc.InnerException != null)
+            if (exc.InnerException != null)
                myErrorTextBox.Text += "Inner Exception: " + exc.InnerException.Message + "\n";
             myErrorTextBox.Text += "\n" + exc.StackTrace;
             myCSharpText.Text = "";
          }
       }
 
-      private string myExtractFromClass (TextReader r)
-      {
+      private string myExtractFromClass(TextReader r) {
          string line = r.ReadLine();
-         while (line != null && line.IndexOf("*Expression*") == -1) 
+         while (line != null && line.IndexOf("*Expression*") == -1)
             line = r.ReadLine();
 
          if (line == null)
-            throw new Exception ("Unexpected EOF");
+            throw new Exception("Unexpected EOF");
 
          r.ReadLine();
 
          string result = "";
          line = r.ReadLine();
-         
-         while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1)
-         {
+
+         while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
             if (line != "")
                result += line.Substring("       ".Length) + "\n";
             line = r.ReadLine();
          }
          if (line == null)
-            throw new Exception ("Unexpected EOF");
+            throw new Exception("Unexpected EOF");
 
          return result;
       }
 
-      public void loadFile (string fileName)
-      {
-         try 
-         {
+      public void loadFile(string fileName) {
+         try {
             TextReader r = new StreamReader(fileName);
-            try 
-            {
+            try {
                myQueryTextBox.Text = myExtractFromClass(r);
-            }
-            finally
-            {
+            } finally {
                r.Close();
             }
             myLastFileName = fileName;
             this.Text = fileName;
-         }
-         catch (Exception exc)
-         {
+         } catch (Exception exc) {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
@@ -243,30 +233,28 @@ namespace BCG
 
       }
 
-      private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code)
-      {
+      private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
 
          myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
-         
-         myCSharpText.Text +="\n/*";
-         myCSharpText.Text +="\n   ----------------------------------------------------------------------";
-         myCSharpText.Text +="\n                               *Expression*";
-         myCSharpText.Text +="\n   ----------------------------------------------------------------------";
+
+         myCSharpText.Text += "\n/*";
+         myCSharpText.Text += "\n   ----------------------------------------------------------------------";
+         myCSharpText.Text += "\n                               *Expression*";
+         myCSharpText.Text += "\n   ----------------------------------------------------------------------";
          foreach (string line in myQueryTextBox.Text.Split('\n'))
-            myCSharpText.Text +="\n       " + line;
-         myCSharpText.Text +="\n\n   ----------------------------------------------------------------------";
-         myCSharpText.Text +="\n                                 *Schema*";
-         myCSharpText.Text +="\n   ----------------------------------------------------------------------";
-         foreach(ColumnInfo ci in schema.columnInfo) 
-            myCSharpText.Text +="\n       " + ci.colName + new string(' ', Math.Max(32 - ci.colName.Length,1) ) + ci.colType.ToString();
-         myCSharpText.Text +="\n*/\n";
-         
+            myCSharpText.Text += "\n       " + line;
+         myCSharpText.Text += "\n\n   ----------------------------------------------------------------------";
+         myCSharpText.Text += "\n                                 *Schema*";
+         myCSharpText.Text += "\n   ----------------------------------------------------------------------";
+         foreach (ColumnInfo ci in schema.columnInfo)
+            myCSharpText.Text += "\n       " + ci.colName + new string(' ', Math.Max(32 - ci.colName.Length, 1)) + ci.colType.ToString();
+         myCSharpText.Text += "\n*/\n";
+
 
          string lastVar = null;
          string variableDeclarations = "";
-         foreach (string var in vars) 
-         {
-            variableDeclarations+= "         ViewStream " + var + ";\n";
+         foreach (string var in vars) {
+            variableDeclarations += "         ViewStream " + var + ";\n";
             lastVar = var;
          }
 
@@ -278,18 +266,18 @@ namespace BCG
          else
             myLastClassName = "MyView";
 
-         myCSharpText.Text +="\n\n   public class " + myLastClassName + " : ViewStream \n   {\n";
-         
+         myCSharpText.Text += "\n\n   public class " + myLastClassName + " : ViewStream \n   {\n";
+
          myCSharpText.Text += "      public " + myLastClassName + "() : this(null) {}\n";
          myCSharpText.Text += "      public " + myLastClassName + "(ViewStreamDatabase db)\n      {\n";
 
          myCSharpText.Text += variableDeclarations;
-         
+
          if (lastVar != null)
             myCSharpText.Text += "\n";
          myCSharpText.Text += code + "\n";
          if (lastVar != null)
-            myCSharpText.Text += "         mySetRealSource(" + lastVar  + ");\n";
+            myCSharpText.Text += "         mySetRealSource(" + lastVar + ");\n";
          myCSharpText.Text += "      }\n";
          myCSharpText.Text += "   }\n";
          myCSharpText.Text += "}";
@@ -310,27 +298,31 @@ namespace BCG
             return Math.Max(header_width, min_width);
          }
       }
-
-      private void myCallback(object o)
-      {
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();
+      // this is the stream callback
+      private void myCallback(object o) {
+         _timer.start();
          DataPacket dp = o as DataPacket;
          myCrono.stop();
          Log.Ui.diag($"Received callback for {dp.schema.tableName}");
-         if (o == null)
-         {
-            System.Windows.Forms.MessageBox.Show (this, "Received a Null Data packet", 
+         if (o == null) {
+            System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
                                                    "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
             return;
          }
          Log.Ui.diag(dp.toIncrementalIDString());
-         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString ("hh:mm:ss:fff tt");
+         myToolStripStatusLabelTime.BackColor = Color.Black;
+         myToolStripStatusLabelRows.BackColor = Color.Black;
+         myToolStripStatusLabelTime.ForeColor = Color.White;
+         myToolStripStatusLabelRows.ForeColor = Color.White;
+         myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
          myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + "  Added = " + dp.addedRows.Count + "  Deleted = " + dp.deletedRows.Count;
          Log.Ui.diag(Text);
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
 
-         myDataGrid.DataSource = new SimpleBindableCollection (dp.newRows.schema, dp.newRows);
-         if (myChangeStyle)
-         {
+         myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
+         var force = false;
+         if (myChangeStyle && force) {
             myChangeStyle = false;
             myDataGrid.Columns.Clear();
             myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
@@ -338,28 +330,30 @@ namespace BCG
          myCrono.start();
          // myDataGrid.CaptionText = myCrono.time.TotalMilliseconds + " ms ";
          // myDataGrid.CaptionText += dp.newRows.Count.ToString () + " records";
-       
+
 
          myDataGrid.HorizScrollBarValue = hscrollval;
 
          var varNames = new List<string>(0);
-         string code = myExp.print(9,varNames);
-         myPrintCode(dp.schema ,varNames, code);
+         string code = myExp.print(9, varNames);
+         myPrintCode(dp.schema, varNames, code);
+         myConsoleRichTextBox.Clear();
+         // this was was very very slow
+         //myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30);
+         _timer.stop();
+         var msecs = _timer.lastIntervalTime;
+         bts.Log.Gen.info($"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {msecs}");
       }
 
-      private Crono  myCrono;
- 
+      private Crono myCrono;
+
       private DataStreamExpression myExp;
 
-      private void myTimer_Tick(object sender, System.EventArgs e)
-      {
-         try 
-         {
-            ViewStreamDatabase.instance.doWork ();
+      private void myTimer_Tick(object sender, System.EventArgs e) {
+         try {
+            ViewStreamDatabase.instance.doWork();
             MasterWorker.getInstance(false).doWork();
-         }
-         catch (Exception exc)
-         {
+         } catch (Exception exc) {
             Log.Ui.warn(exc.ToString());
 
             myDataGrid.DataSource = null;
@@ -370,27 +364,22 @@ namespace BCG
          }
       }
 
-      class StringWrapper
-      {
-         public StringWrapper(string s)
-         {
+      class StringWrapper {
+         public StringWrapper(string s) {
             myTableName = s;
          }
-         public string tableName 
-         {
-            get{return myTableName;}
+         public string tableName {
+            get { return myTableName; }
          }
          private string myTableName;
       }
 
-      private void MainForm_Closed(object sender, System.EventArgs e)
-      {
+      private void MainForm_Closed(object sender, System.EventArgs e) {
          if (myExp != null)
             myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
       }
 
-      private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
          myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
          myQueryTextBox.SelectionColor = Color.Black;
@@ -398,49 +387,44 @@ namespace BCG
          myTryExpression(myQueryTextBox.Text);
       }
 
-      private void myToolStripMenuItemExecSel_Click (object sender, EventArgs e)
-      {
-         int selstart = myQueryTextBox.SelectionStart;
-         int sellength = myQueryTextBox.SelectionLength;
-         myQueryTextBox.SelectionStart = 0;
-         myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
-         myQueryTextBox.SelectionColor = Color.Black;
-         myQueryTextBox.SelectionStart = selstart;
-         myQueryTextBox.SelectionLength = sellength;
-         myQueryTextBox.SelectionColor = Color.Crimson;
-         myTryExpression (myQueryTextBox.SelectedText);
-         myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
-         myQueryTextBox.SelectionLength = 0;
-         myQueryTextBox.SelectionColor = Color.Black;
+      private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
+         // int selstart = myQueryTextBox.SelectionStart;
+         // int sellength = myQueryTextBox.SelectionLength;
+         // myQueryTextBox.SelectionStart = 0;
+         // myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
+         // myQueryTextBox.SelectionColor = Color.White;
+         // myQueryTextBox.SelectionStart = selstart;
+         // myQueryTextBox.SelectionLength = sellength;
+         // myQueryTextBox.SelectionColor = Color.Crimson;
+         // myTryExpression(myQueryTextBox.SelectedText);
+         // myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
+         // myQueryTextBox.SelectionLength = 0;
+         // myQueryTextBox.SelectionColor = Color.White;
       }
 
-      private void myMenuItemOpenClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemOpenClick(object sender, System.EventArgs e) {
          OpenFileDialog od = new OpenFileDialog();
          od.Filter = "C# class (*.cs)|*.cs";
 
-         if(od.ShowDialog() == DialogResult.OK)
+         if (od.ShowDialog() == DialogResult.OK)
             loadFile(od.FileName);
       }
 
-      private void mySave(string fileName)
-      {
+      private void mySave(string fileName) {
          using (TextWriter w = new StreamWriter(fileName))
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
       }
 
-      private void myMenuItemSaveClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemSaveClick(object sender, System.EventArgs e) {
          if (myLastFileName == null)
-            myMenuItemSaveAsClick(sender,e);
+            myMenuItemSaveAsClick(sender, e);
          else
             mySave(myLastFileName);
       }
 
-      private void myMenuItemSaveAsClick(object sender, System.EventArgs e)
-      {
+      private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
          SaveFileDialog sd = new SaveFileDialog();
          if (myLastFileName != null)
             sd.FileName = myLastFileName;
@@ -448,62 +432,53 @@ namespace BCG
             sd.FileName = myLastClassName + ".cs";
 
          if (sd.ShowDialog() == DialogResult.OK)
-            mySave (sd.FileName);
+            mySave(sd.FileName);
       }
 
-      private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
-      {
-         if (myTableListBox.SelectedItem != null) 
-         {
-            string tableName = myTableListBox.SelectedItem.ToString ();
+      private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
+         if (myTableListBox.SelectedItem != null) {
+            string tableName = myTableListBox.SelectedItem.ToString();
             myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
-            myTryExpression (myQueryTextBox.Text);
+            myTryExpression(myQueryTextBox.Text);
          }
       }
 
-      private void myDataGrid_MouseEnter(object sender, System.EventArgs e)
-      {
+      private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
          //myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
          // if (myDataGrid.TableStyles.Count > 0)
          //    myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
       }
 
-      private void myDataGrid_MouseLeave(object sender, System.EventArgs e)
-      {
+      private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
          //myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
          // if (myDataGrid.TableStyles.Count > 0)
          //    myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
       }
 
-      private void myCSharpText_MouseEnter(object sender, System.EventArgs e)
-      {
-         myCSharpText.BackColor = Color.Gainsboro;
-         myCSharpText.Font = new Font (myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
+      private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
+         //myCSharpText.BackColor = Color.Gainsboro;
+         myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
       }
 
-      private void myCSharpText_MouseLeave(object sender, System.EventArgs e)
-      {
-         myCSharpText.BackColor = Color.White;
-         myCSharpText.Font = new Font (myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
+      private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
+         //myCSharpText.BackColor = Color.White;
+         myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
       }
 
-      private void myTableListBox_MouseEnter(object sender, System.EventArgs e)
-      {
-         myTableListBox.BackColor = Color.Gainsboro;
-         myTableListBox.Font = new Font (myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
+      private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
+         //myTableListBox.BackColor = Color.Gainsboro;
+         myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
       }
 
-      private void myTableListBox_MouseLeave(object sender, System.EventArgs e)
-      {
-         myTableListBox.BackColor = Color.White;
-         myTableListBox.Font = new Font (myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
+      private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
+         //myTableListBox.BackColor = Color.White;
+         myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
       }
 
-      private void myLoad (object sender, EventArgs e)
-      {         
-         myFreezeView (false);
+      private void myLoad(object sender, EventArgs e) {
+         myFreezeView(false);
 
-         myToolStripMenuItemSite.DropDownItems.Clear ();
+         myToolStripMenuItemSite.DropDownItems.Clear();
          // foreach (var name in SitesCfg.instance.siteNames)
          // {
          //    ToolStripMenuItem mi = new ToolStripMenuItem (name);
@@ -517,144 +492,123 @@ namespace BCG
          // }
 
          myConsoleRichTextBox.ReadOnly = true;
-         myConsoleOutput = new RichTextBoxStreamWriter (myConsoleRichTextBox);
-         Console.SetOut (myConsoleOutput);
-         Console.SetError (myConsoleOutput);
-         if(BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-11, myConsoleRichTextBox.Handle) == 0) // set stdout
+         myConsoleOutput = new RichTextBoxStreamWriter(myConsoleRichTextBox);
+         Console.SetOut(myConsoleOutput);
+         Console.SetError(myConsoleOutput);
+         if (BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-11, myConsoleRichTextBox.Handle) == 0) // set stdout
          {
-            var ex = new Win32Exception ();  // same as new Win32Exception(Marshal.GetLastWin32Error())
-            Console.WriteLine (ex.Message);
-         }
-         else
-         {
-            Console.WriteLine ("Set stdout worked.");
+            var ex = new Win32Exception();  // same as new Win32Exception(Marshal.GetLastWin32Error())
+            Console.WriteLine(ex.Message);
+         } else {
+            Console.WriteLine("Set stdout worked.");
          }
 
-         if(BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-12, myConsoleRichTextBox.Handle) == 0) // set stderr
+         if (BCG.Utilities.Win32.WindowsAPI.SetStdHandle(-12, myConsoleRichTextBox.Handle) == 0) // set stderr
          {
-            var ex = new Win32Exception ();  // same as new Win32Exception(Marshal.GetLastWin32Error())
-            Console.WriteLine (ex.Message);
+            var ex = new Win32Exception();  // same as new Win32Exception(Marshal.GetLastWin32Error())
+            Console.WriteLine(ex.Message);
+         } else {
+            Console.WriteLine("Set stderr worked.");
          }
-         else
-         {
-            Console.WriteLine ("Set stderr worked.");
-         }
-         Console.WriteLine ("Redirecting console ouput to this window...");
+         Console.WriteLine("Redirecting console ouput to this window...");
 
-         myLoadAssemblies ();
+         myLoadAssemblies();
 
-         ViewStreamDatabase.instance = new ViewStreamDatabase ();
-         myAllTagsStream = myGetAllTagsStream ();
-         myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate (myHandleTableListChange);
+         ViewStreamDatabase.instance = new ViewStreamDatabase();
+         myAllTagsStream = myGetAllTagsStream();
+         myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
          myTabControl.SelectedIndex = 3;
-         myTreeView.AfterSelect += new TreeViewEventHandler (myHandleTreeNodeSelect);
+         myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
          //do this lazily so we don't have to have bps, etc up to run dsqa
          //_library = new LibraryForm(this);
       }
 
-      private void myMenuItemSiteClick (object sender, System.EventArgs e)
-      {
-         ToolStripMenuItem mi = (ToolStripMenuItem) sender;
+      private void myMenuItemSiteClick(object sender, System.EventArgs e) {
+         ToolStripMenuItem mi = (ToolStripMenuItem)sender;
 
-         if (mi.Text != mySite)
-         {
+         if (mi.Text != mySite) {
             mySite = mi.Text;
             foreach (var m in mySiteItems)
                m.Checked = false;
             mi.Checked = true;
 
-            myAllTagsStream.onChange -= new BCG.DataStream.ChangeDelegate (myHandleTableListChange);
-            myTableListBox.Items.Clear ();
+            myAllTagsStream.onChange -= new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
+            myTableListBox.Items.Clear();
 
             // SitesCore.setMySiteName (mySite);
-            
-            ViewStreamDatabase.instance = new ViewStreamDatabase ();
-            myAllTagsStream = myGetAllTagsStream ();
-            myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate (myHandleTableListChange);
+
+            ViewStreamDatabase.instance = new ViewStreamDatabase();
+            myAllTagsStream = myGetAllTagsStream();
+            myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
 
             myDataGrid.DataSource = null;
             myTabControl.SelectedIndex = 3;
-            mySetText ();
+            mySetText();
          }
       }
 
-      private void myClosing (object sender, CancelEventArgs e)
-      {
+      private void myClosing(object sender, CancelEventArgs e) {
       }
 
-      private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e)
-      {
-         if (myTreeView.SelectedNode != null)
-         {
-            if (myTreeView.SelectedNode.LastNode == null)
-            {
+      private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
+         if (myTreeView.SelectedNode != null) {
+            if (myTreeView.SelectedNode.LastNode == null) {
                string sn = myTreeView.SelectedNode.Parent.Text + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
-      private void myToolStripMenuItemExit_Click (object sender, EventArgs e)
-      {
-         Close ();
+      private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
+         Close();
       }
 
-      private void myToolStripMenuItemFreeze_Click (object sender, EventArgs e)
-      {
-         myFreezeView (myTimer.Enabled);
+      private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
+         myFreezeView(myTimer.Enabled);
       }
 
-      private void myFreezeView (bool freeze)
-      {
+      private void myFreezeView(bool freeze) {
          myTimer.Enabled = !freeze;
          myToolStripMenuItemFreeze.Checked = freeze;
          myToolStripMenuItemStep.Enabled = freeze;
       }
 
-      private void myToolStripMenuItemStep_Click (object sender, EventArgs e)
-      {
-         ViewStreamDatabase.instance.doWork ();
+      private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
+         ViewStreamDatabase.instance.doWork();
       }
 
-      private void myToolStripMenuItemLoadAss_Click (object sender, EventArgs e)
-      {
-         OpenFileDialog ofd = new OpenFileDialog ();
+      private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
+         OpenFileDialog ofd = new OpenFileDialog();
          ofd.ShowReadOnly = true;
-         DialogResult dr = ofd.ShowDialog (this);
-         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
-         {
-            PrimitiveManager.instance.loadAssembly (ofd.FileName);
-            myLoadAssemblies ();
+         DialogResult dr = ofd.ShowDialog(this);
+         if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
+            PrimitiveManager.instance.loadAssembly(ofd.FileName);
+            myLoadAssemblies();
          }
       }
 
-      private void myToolStripMenuItemTest_Click (object sender, EventArgs e)
-      {
-         if (myCurrentStream != null)
-         {
-            myCurrentStream.onChange -= new DataStream.ChangeDelegate (myCallback);
-            myCurrentStream.onChange += new DataStream.ChangeDelegate (myCallback);
+      private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
+         if (myCurrentStream != null) {
+            myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
+            myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
          }
       }
 
-      private void myToolStripMenuItemClearConsole_Click (object sender, EventArgs e)
-      {
-         myConsoleRichTextBox.Clear ();
-         Console.WriteLine ("Redirecting console ouput to this window...");
+      private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
+         myConsoleRichTextBox.Clear();
+         Console.WriteLine("Redirecting console ouput to this window...");
       }
 
-      private void myTableFilterBox_TextChanged(object sender, EventArgs e)
-      {                     
+      private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
          myUpdateTableList();
       }
 
-      private void myUpdateTableList()
-      {
+      private void myUpdateTableList() {
          IBlueRowCollection rows = myAllTagsStream.rows;
          if (rows == null)
             return;
-         
-      
+
+
          ICollection oldItems = myTableListBox.Items;
 
          myTableListBox.BeginUpdate();
@@ -664,15 +618,12 @@ namespace BCG
          myTableListBox.Items.Clear();
          SortedList<string, bool> sl = new SortedList<string, bool>();
          if (string.IsNullOrEmpty(myTableFilterBox.Text))
-            foreach (IBlueRow r in rows)
-            {
+            foreach (IBlueRow r in rows) {
                sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true;
             }
-         else
-         {
+         else {
             Regex tableRegex = new Regex(myTableFilterBox.Text);
-            foreach (IBlueRow r in rows)
-            {
+            foreach (IBlueRow r in rows) {
                string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length);
                if (tableRegex.IsMatch(thisTable))
                   sl[thisTable] = true;
@@ -687,11 +638,10 @@ namespace BCG
             myTableListBox.TopIndex = top;
          else
             myTableListBox.TopIndex = Math.Min(sl.Count - 1, topindex);
-         myTableListBox.EndUpdate();         
+         myTableListBox.EndUpdate();
       }
 
-      private void testGridToolStripMenuItem_Click(object sender, EventArgs e)
-      {
+      private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
          HighPerformanceTimer timer = new HighPerformanceTimer();
          //while (true)
          //{
@@ -713,11 +663,9 @@ namespace BCG
          foreach (IBlueRow r in s.rows)
             rows.Add(r);
 
-         for (int i = 0; i < 100; i++)
-         {
+         for (int i = 0; i < 100; i++) {
             timer.start();
-            rows.Sort(delegate(IBlueRow r1, IBlueRow r2)
-            {
+            rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
                return _sort_dicts_index(r1, r2, 5, true);
             });
             timer.stop();
@@ -725,22 +673,19 @@ namespace BCG
          }
       }
 
-      private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high)
-      {
+      private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
          if (r1.Contains(column) && r2.Contains(column))
             return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
          else
             return 0;
       }
 
-      private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high)
-      {
+      private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
          //return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
          return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
       }
 
-      internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high)
-      {
+      internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
          if (myIsDontCare(o1))
             return myIsDontCare(o2) ? 0 : 1;
          else
@@ -748,16 +693,14 @@ namespace BCG
                                                     : ((IComparable)o2).CompareTo(o1));
       }
 
-      private static bool myIsDontCare(object o)
-      {
+      private static bool myIsDontCare(object o) {
          return o == null || o == DBNull.Value ||
                   o is string && String.IsNullOrEmpty((string)o) ||
                   o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
                   o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
       }
 
-      private void _new_grid_form(object args)
-      {
+      private void _new_grid_form(object args) {
          //System.Timers.Timer t = new System.Timers.Timer();
          ////t.SynchronizingObject
          //BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
@@ -768,5 +711,13 @@ namespace BCG
       }
 
       private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
+
+      private void myTableFilterLabel_Click(object sender, EventArgs e) {
+
+      }
+
+      private void MainForm_Load(object sender, EventArgs e) {
+
+      }
    }
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.resx

@@ -1,64 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!-- 
-    Microsoft ResX Schema 
-    
-    Version 2.0
-    
-    The primary goals of this format is to allow a simple XML format 
-    that is mostly human readable. The generation and parsing of the 
-    various data types are done through the TypeConverter classes 
-    associated with the data types.
-    
-    Example:
-    
-    ... ado.net/XML headers & schema ...
-    <resheader name="resmimetype">text/microsoft-resx</resheader>
-    <resheader name="version">2.0</resheader>
-    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
-    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
-    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
-    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
-    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
-        <value>[base64 mime encoded serialized .NET Framework object]</value>
-    </data>
-    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
-        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
-        <comment>This is a comment</comment>
-    </data>
-                
-    There are any number of "resheader" rows that contain simple 
-    name/value pairs.
-    
-    Each data row contains a name, and value. The row also contains a 
-    type or mimetype. Type corresponds to a .NET class that support 
-    text/value conversion through the TypeConverter architecture. 
-    Classes that don't support this are serialized and stored with the 
-    mimetype set.
-    
-    The mimetype is used for serialized objects, and tells the 
-    ResXResourceReader how to depersist the object. This is currently not 
-    extensible. For a given mimetype the value must be set accordingly:
-    
-    Note - application/x-microsoft.net.object.binary.base64 is the format 
-    that the ResXResourceWriter will generate, however the reader can 
-    read any of the formats listed below.
-    
-    mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with 
-            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
-            : and then encoded with base64 encoding.
-    
-    mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with 
-            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
-            : and then encoded with base64 encoding.
-
-    mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array 
-            : using a System.ComponentModel.TypeConverter
-            : and then encoded with base64 encoding.
-    -->
   <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
     <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
     <xsd:element name="root" msdata:IsDataSet="true">

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -1,24 +1,35 @@
-using System;
-using System.Windows.Forms;
 using BCG;
-using System.IO;
-using System.Runtime.Serialization;
-using System.Runtime.Serialization.Formatters.Binary;
-using bts;
-using mbts;
-using mbts.config;
+
 using bts.utils;
 
+using mbts.config;
+
+using System;
+using System.Linq;
+using System.Windows.Forms;
+using System.Threading.Tasks;
+using System.ComponentModel;
+
 static class Program {
-   [STAThread]
-   static void Main(string[] args) {
+   static void init(string[] args) {
+      Func<string, string, Task> login = async (l, p) => {
+         var login = l;
+         var passwd = p;
+         var service_ctx = bts.api.core.ServiceContext.create();
+         await service_ctx.login(login, passwd);
+      };
       BTSNativeMethods.provide_command_line(args, args.Length);
       Configuration.initialize();
-      bts.Log.Gen.info($"Starting DSQA");
-      Console.WriteLine("Starting DataStreamQueryAnalyzer...");
-      Console.WriteLine("Starting Test Publisher");
-      // TestPubliser.publish();
-      // TestPubliser.subscribe();
+      Task.Run(async () => await login("bts", String.Empty));
+   }
+   [STAThread]
+   static void Main(string[] args) {
+      init(args);
+      string args_concat = String.Empty;
+      args.ToList().ForEach(t => args_concat += t);
+      var init_log = $"Starting DSQA args: {args_concat} len: {args.Length}";
+      bts.Log.Gen.info(init_log);
+      Console.WriteLine(init_log);
       Console.WriteLine("Launching Main Window");
       MainForm mainForm = new MainForm();
       if (args.Length > 0) {
@@ -27,65 +38,3 @@ static class Program {
       Application.Run(mainForm);
    }
 }
-
-class TestPubItem : IFormatter, IPubItem
-{
-   public DateTime send_time = DateTime.Now;
-   public object Deserialize(Stream str)
-   {
-      BinaryReader br = new BinaryReader(str);
-      var d = br.ReadDouble();
-      TestPubItem rv = new TestPubItem();
-      rv.send_time = DateTime.FromOADate(d);
-      return rv;
-   }
-   public void Serialize(Stream str, Object obj)
-   {
-      TestPubItem tpi = obj as TestPubItem;
-      BinaryWriter bw = new BinaryWriter(str);
-      bw.Write(tpi.send_time.ToOADate());
-   }
-   public System.Runtime.Serialization.SerializationBinder? Binder { get; set; }
-
-   public System.Runtime.Serialization.StreamingContext Context { get; set; }
-
-   public System.Runtime.Serialization.ISurrogateSelector? SurrogateSelector { get; set; }
-
-   public string getTag()
-   {
-      return "Test_Tag";
-   }
-}
-
-class TestPubliser
-{
-   public static void publish()
-   {
-      int n_pubs = 1000;
-      Publisher<TestPubItem> publisher = new Publisher<TestPubItem>();
-      for (int i = 0; i < n_pubs; i++)
-      {
-         TestPubItem pi = new TestPubItem();
-         pi.send_time = DateTime.Now;
-         bts.Log.Gen.info($"DSQA publishing tag: {pi.getTag()}");
-         publisher.publish(pi);
-         System.Threading.Thread.Sleep(100);
-         MasterWorker.getInstance(false).doWork();
-      }
-   }
-   public static void subscribe()
-   {
-      ManagedSubscriber ms = new ManagedSubscriber();
-      ms.subscribe("Test_Tag", new Handler(on_data));
-      while (true)
-      {
-         ms.doWork();
-      }
-   }
-   public static void on_data(IPubItem pi)
-   {
-      Console.WriteLine("Received data");
-   }
-}
-
-

src/DataStreamSuite/DataStreamSuite.sln

@@ -1,4 +1,3 @@
-
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31903.59
@@ -35,6 +34,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "Col
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "ExpressionCore\ExpressionCore.csproj", "{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStreamTests\DataStreamTests.csproj", "{66AB10D3-155F-4663-B1BC-D0818E17B87B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "SharedStreams\SharedStreams.csproj", "{C1EBC423-4668-43D4-B7E5-A291086BA823}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -80,6 +83,10 @@ Global
 		{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Debug|Any CPU.Build.0 = Debug|x64
 		{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.ActiveCfg = Release|x64
 		{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.Build.0 = Release|x64
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -108,5 +115,9 @@ Global
 		{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C1EBC423-4668-43D4-B7E5-A291086BA823}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="NUnit" Version="3.13.3" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
+    <PackageReference Include="NUnit.Analyzers" Version="3.5.0" />
+    <PackageReference Include="coverlet.collector" Version="3.1.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\pubsub\pubsub.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+  </ItemGroup>
+
+
+</Project>

src/DataStreamSuite/DataStreamTests/StreamSerialization/UniversalSerializationTest.cs

@@ -0,0 +1,195 @@
+using System;
+using System.IO;
+
+using NUnit.Framework;
+
+namespace BCG {
+
+   /// <summary>
+   /// Tests the deserialization happening within the pubsub.
+   /// </summary>
+   public class UniversalSerializationTest {
+      protected IBlueTableSchema mySchema;
+      protected BlueTable myTable;
+      protected IBlueDataSerializer mySer = new BlueDataSerializationController();
+      protected IPubsubDeserializer myPubsubDeser = new PubsubDeserializationController();
+      protected IClientDeserializer myClientDeser = new ClientDeserializationController();
+
+      public UniversalSerializationTest() {
+         mySchema = new BlueTableSchema(myGetTableName(),
+                                          new ColumnInfo("id", 0, typeof(int)),
+                                          new ColumnInfo("i1", 1, typeof(int)),
+                                          new ColumnInfo("d1", 2, typeof(double)),
+                                          new ColumnInfo("dt", 3, typeof(DateTime)),
+                                          new ColumnInfo("x", 4, typeof(byte))
+            );
+      }
+
+      protected virtual string myGetTableName() {
+         return "testUniversal";
+      }
+
+      /// <summary>
+      /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
+      /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer. 
+      /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
+      /// through the clientDeserializer to get all the rows.
+      /// 
+      /// </summary>
+      [Test]
+      public void testEntireProcess() {
+         myTable = new BlueTable(mySchema);
+         byte[] data;
+         int id;
+         //create
+         for (id = 0; id < 100; id++)
+            myTable.addRow(myGetRow(id));
+
+         byte[] ops;
+         Array rows;
+         IBlueRowCollection del, add;
+         IBlueRowCollection[] receivedData;
+         IBlueRow r;
+
+         //create
+         mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
+         myWrite(StreamOpCode.create, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.create, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+
+         BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //single add
+         r = myGetRow(id);
+         BlueTable temp = new BlueTable(mySchema);
+         myTable.addRow(r);
+         temp.addRow(r);
+         mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
+         myWrite(StreamOpCode.add, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.add, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //single delete
+         r = myTable[0];   //delete row with id 0
+         temp = new BlueTable(mySchema);
+         myTable.deleteRow(0);
+         temp.addRow(r);
+         mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
+         myWrite(StreamOpCode.remove, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.remove, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //batch deletes and adds
+         //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
+         BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
+         for (int i = 1; i < 25; i++) {
+            delRows.addRow(myTable[i]);
+            myTable.deleteRow(i);
+         }
+         for (id = 101; id < 125; id++) {
+            r = myGetRow(id);
+            addRows.addRow(r);
+            myTable.addRow(r);
+         }
+         for (int i = 25; i < 125; i++)
+            Assert.AreEqual(true, myTable.containsID(i));
+         Assert.AreEqual(100, myTable.Count);
+
+         mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
+         myWrite(StreamOpCode.multipleRemoveAdd, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(2, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+         BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //pure changes only
+         //change the first 25, 25-49
+         BlueTable changedRows = new BlueTable(mySchema);
+         for (int i = 25; i < 50; i++) {
+            MutableBlueRow mr = new MutableBlueRow(myTable[i]);
+            mr.setValue(4, Byte.MaxValue);
+            myTable.changeRow(mr);
+            changedRows.addRow(mr);
+         }
+         mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
+         myWrite(StreamOpCode.multipleChange, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //final test
+         data = myPubsubDeser.getState();
+         new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.create, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+      }
+
+      private IBlueRow myGetRow(int id) {
+         return new BlueRow(mySchema,                        // the day pp fixed this test. 
+                              id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
+      }
+
+      private void myWrite(byte op, byte[] data) {
+         string s = "";
+         if (op == StreamOpCode.add)
+            s = "add one row";
+         else if (op == StreamOpCode.change)
+            s = "change one row";
+         else if (op == StreamOpCode.create)
+            s = "create";
+         else if (op == StreamOpCode.multipleChange)
+            s = "multiple change";
+         else if (op == StreamOpCode.multipleRemoveAdd)
+            s = "multiple add remove";
+         else if (op == StreamOpCode.remove)
+            s = "remove one row";
+         Console.WriteLine("yes, here , op = {0}, size = {1}", s, data.Length);
+      }
+   }
+}

src/DataStreamSuite/DataStreamTests/Usings.cs

@@ -0,0 +1 @@
+global using NUnit.Framework;

src/DataStreamSuite/PubSub/AllPublishedTagState.cs

@@ -4,15 +4,13 @@ using System.Collections;
 using System.Collections.Generic;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Holds the state for a single tag. If the tagType has an
    /// IPubSubServerTagHandler, it is updated and the state stored
    /// there. Otherwise, this class saves the last published data.
    /// </summary>
-   internal class SingleTagHandler
-   {
+   internal class SingleTagHandler {
       private IPubSubServerTagHandler myHandler = null;
       private string myTagType, myTag;
       /// <summary>
@@ -20,9 +18,7 @@ namespace BCG
       /// </summary>
       private byte[] myData = null;
 
-      public SingleTagHandler(string tag, string tagType)
-      {
-         
+      public SingleTagHandler(string tag, string tagType) {
          myTagType = tagType;
          myTag = tag;
          myHandler = TagType2HandlerMap.getHandler(tagType, false);
@@ -30,23 +26,18 @@ namespace BCG
 
       public string tagType { get { return myTagType; } }
 
-      public void publish(Stream data)
-      {
+      public void publish(Stream data) {
          long pos = data.Position;
-         if (myHandler != null)
-         {
+         if (myHandler != null) {
             myHandler.update(myTag, new Stream[] { data });
-         }
-         else
-         {
+         } else {
             myData = new byte[data.Length];
             data.Read(myData, 0, myData.Length);
          }
          data.Position = pos;
       }
 
-      public byte[] getContent()
-      {
+      public byte[] getContent() {
          return myHandler == null ? myData : myHandler.getContent(myTag);
       }
    }
@@ -54,46 +45,37 @@ namespace BCG
    /// <summary>
    /// Keeps the state of all the tags published.
    /// </summary>
-   public class AllPublishedTagState
-   {
+   public class AllPublishedTagState {
       private Dictionary<string, SingleTagHandler> myTagHandler = new Dictionary<string, SingleTagHandler>();
 
       public AllPublishedTagState() { }
 
-      public void publish(string tag, string tagType, Stream data)
-      {
+      public void publish(string tag, string tagType, Stream data) {
          if (!myTagHandler.ContainsKey(tag))
             myTagHandler.Add(tag, new SingleTagHandler(tag, tagType));
 
          myTagHandler[tag].publish(data);
       }
 
-      public void publish(string tag, string tagType, byte[] data)
-      {
+      public void publish(string tag, string tagType, byte[] data) {
          publish(tag, tagType, new MemoryStream(data));
       }
 
-      public string[] publishedTags
-      { get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } }
+      public string[] publishedTags { get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } }
 
-      public byte[] getContent(string tag, out string tagType)
-      {
+      public byte[] getContent(string tag, out string tagType) {
          tagType = myTagHandler[tag].tagType;
          return myTagHandler[tag].getContent();
       }
 
-      public bool contains(string tag)
-      {
-          return myTagHandler.ContainsKey(tag);
+      public bool contains(string tag) {
+         return myTagHandler.ContainsKey(tag);
       }
-      public void clear(string tag)
-      {
-         if (myTagHandler.ContainsKey(tag))
-         {
+      public void clear(string tag) {
+         if (myTagHandler.ContainsKey(tag)) {
             myTagHandler.Remove(tag);
-         }
-         else{}
-            // BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning);
+         } else { }
+         // BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning);
 
       }
    }

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/PubSubServerUnmanagedTCPConnection.cs

@@ -1,117 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Runtime.InteropServices;
-using mbts;
-//using BCG.Utilities;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Runtime.InteropServices;
+// using mbts;
+// //using BCG.Utilities;
 
-namespace BCG
-{
-   public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection
-   {
-      private BlutoSubAddress myServerAddress;
-      private SmartConnectionManagedWrapper myWrapper;
-      private ConnectionStatusChange myConnCallback;
-      private SyncQueue<Pair<int, byte[]>> myAllMsgsQueue = new SyncQueue<Pair<int, byte[]>>();
+// namespace BCG
+// {
+//    public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection
+//    {
+//       private BlutoSubAddress myServerAddress;
+//       private SmartConnectionManagedWrapper myWrapper;
+//       private ConnectionStatusChange myConnCallback;
+//       private SyncQueue<Pair<int, byte[]>> myAllMsgsQueue = new SyncQueue<Pair<int, byte[]>>();
 
-      public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper)
-      {
-         myServerAddress = new BlutoSubAddress(serverURL);
-         myWrapper = wrapper;
-         myWrapper.initConnection(new BlutoSubAddress(serverURL));
-         myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages));
-      }
+//       public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper)
+//       {
+//          myServerAddress = new BlutoSubAddress(serverURL);
+//          myWrapper = wrapper;
+//          myWrapper.initConnection(new BlutoSubAddress(serverURL));
+//          myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages));
+//       }
 
-      #region IPubSubServerConnection Members
+//       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string confId)
-      {
-         //BlueEventLog.logAssert(false, "This function should not be used on this class");
-      }
+//       public void sendWithConfirmation(byte[] data, ref string confId)
+//       {
+//          //BlueEventLog.logAssert(false, "This function should not be used on this class");
+//       }
 
-      public void send(byte[] data)
-      {
-         myWrapper.send(myServerAddress, data);
-      }
+//       public void send(byte[] data)
+//       {
+//          myWrapper.send(myServerAddress, data);
+//       }
 
-      public void send(System.IO.Stream str)
-      {
-         send (StreamUtilityMethods.getBytesFromStream(str));
-      }
+//       public void send(System.IO.Stream str)
+//       {
+//          send (StreamUtilityMethods.getBytesFromStream(str));
+//       }
 
-      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
-      {
-         List<Pair<int, byte[]>> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements);
-         List<byte[]> rv = new List<byte[]>();
-         bool connSet = false;
-         foreach (Pair<int, byte[]> p in msgs)
-         {
-            int msgType = p.first;
-            byte[] msg = p.second;
-            if (msgType == 1)
-            {
-               rv.Add(msg);
-            }
-            else
-            {
-               connSet = true;
-               string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established");
-               Console.WriteLine(connMsg);
-               myConnCallback(myServerAddress.ToString(), msgType == 3);
-            }
-         }
-         return connSet ? new List<byte[]>() : rv;
-      }
-         
-      public string serverURL
-      {
-         get { return myServerAddress.ToString(); }
-      }
+//       public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
+//       {
+//          List<Pair<int, byte[]>> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements);
+//          List<byte[]> rv = new List<byte[]>();
+//          bool connSet = false;
+//          foreach (Pair<int, byte[]> p in msgs)
+//          {
+//             int msgType = p.first;
+//             byte[] msg = p.second;
+//             if (msgType == 1)
+//             {
+//                rv.Add(msg);
+//             }
+//             else
+//             {
+//                connSet = true;
+//                string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established");
+//                Console.WriteLine(connMsg);
+//                myConnCallback(myServerAddress.ToString(), msgType == 3);
+//             }
+//          }
+//          return connSet ? new List<byte[]>() : rv;
+//       }
 
-      public string clientURL
-      {
-         get { return myWrapper.getClientAsSeenByServer (serverURL); }
-      }
+//       public string serverURL
+//       {
+//          get { return myServerAddress.ToString(); }
+//       }
 
-      public void registerCallback(ConnectionStatusChange del)
-      {
-         myConnCallback = del;
-      }
+//       public string clientURL
+//       {
+//          get { return myWrapper.getClientAsSeenByServer (serverURL); }
+//       }
 
-      public void registerDataCallback(DataDelegate del)
-      {
-      }
+//       public void registerCallback(ConnectionStatusChange del)
+//       {
+//          myConnCallback = del;
+//       }
 
-      public void registerSendConfirmationCallback(SendConfirmationDelegate del)
-      {
-      }
+//       public void registerDataCallback(DataDelegate del)
+//       {
+//       }
 
-      #endregion
+//       public void registerSendConfirmationCallback(SendConfirmationDelegate del)
+//       {
+//       }
 
-      #region IDisposable Members
+//       #endregion
 
-      public void Dispose()
-      {
-         
-      }
+//       #region IDisposable Members
 
-      #endregion
+//       public void Dispose()
+//       {
 
-      #region IWorker Members
-      public void doWork()
-      {
-      }
-      #endregion
+//       }
 
-      /// <summary>
-      /// this is the only method that it executed on
-      /// a different thread
-      /// </summary>
-      /// <param name="data"></param>
-      /// <returns></returns>
-      private void myHandleServerMessages(int msgType, byte[] msg)
-      {
-         myAllMsgsQueue.enqueue(new Pair<int, byte[]>(msgType, msg));
-      }
-   }
-}
+//       #endregion
+
+//       #region IWorker Members
+//       public void doWork()
+//       {
+//       }
+//       #endregion
+
+//       /// <summary>
+//       /// this is the only method that it executed on
+//       /// a different thread
+//       /// </summary>
+//       /// <param name="data"></param>
+//       /// <returns></returns>
+//       private void myHandleServerMessages(int msgType, byte[] msg)
+//       {
+//          myAllMsgsQueue.enqueue(new Pair<int, byte[]>(msgType, msg));
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs

@@ -4,8 +4,7 @@ using System.Text;
 using System.IO;
 using BCG.Utilities.General;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// This is the interface that both the PublisherCore and the
    /// SubscriberCore use to communicate with the PubsubServer.
@@ -20,22 +19,23 @@ namespace BCG
    /// multiple threads need to communicate to the PubSub, they 
    /// should create their own Subscribers.
    /// </summary>
-   public interface IPubSubServerConnection : IDisposable, IWorker
-   {
+   public interface IPubSubServerConnection : IDisposable, IWorker {
       /// <summary>
       /// The delegate is fired signalling if the data send was
       /// successful or not
       /// </summary>
-      void sendWithConfirmation(byte[] data, ref string confId);
+      //void sendWithConfirmation(byte[] data, ref string confId);
       /// <summary>
       /// send is a non waiting call with no guarantee
       /// of data being delivered
       /// </summary>
-      void send (byte[] data);
-      /// <summary>
-      /// similiar to above
-      /// </summary>
+      // void send(byte[] data);
+      // /// <summary>
+      // /// similiar to above
+      // /// </summary>
       void send(Stream str);
+      void send(PubSubMessage msg);
+      void send_with_confirmation(PubSubMessage msg, ref string confId);
       /// <summary>
       /// Returns with 0 or more elements, iff the timeout
       /// expires or maxElements have been reached (selective polling),
@@ -51,13 +51,13 @@ namespace BCG
       /// <summary>
       /// server URL to which the connection is made
       /// </summary>
-      string serverURL { get;}
+      string serverURL { get; }
       /// <summary>
       /// client URL as seen by the server. if a connection has not
       /// been established (as can happen in TCP) an empty string 
       /// is returned
       /// </summary>
-      string clientURL { get;}
+      string clientURL { get; }
 
       /// <summary>
       /// registers the callback to receive 

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -9,8 +9,7 @@ using System.IO;
 using System.Threading;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
    /// from the server, and in case of missing hbs, informs its clients, and tries to
@@ -23,91 +22,94 @@ namespace BCG
    /// </summary>
    public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
                                                    ITCPMessageHandler,
-                                                   ITCPConnectionHandler
-   {
-      private ITCPConnection myConnection, myTempNewConnection;
-      private BlutoSubAddress myServerAddress;
-      private ThreadSafeQueue<Pair<string, bool>> myConfs = new ThreadSafeQueue<Pair<string, bool>>();
-      private SyncQueue<byte[]> myDataQueue = new SyncQueue<byte[]>();
-      private DateTime myLastHB = DateTime.Now;
-      private ConnectionStatusChange myConnCallback;
-      private DataDelegate myDataCallback;
-      private SendConfirmationDelegate myOnSendConfirmCallback;
-      private TimeSpan myConnectionDropThreshold =
+                                                   ITCPConnectionHandler {
+      private ITCPConnection _connection, _new_connection;
+      private readonly BlutoSubAddress _server_address;
+      // if a message(with confirm) can not be sent , its id is queued here, 
+      // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
+      //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
+      private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
+      // in onData, data is queued here
+      private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
+      private DateTime _last_hb = DateTime.Now;
+      private ConnectionStatusChange _conn_callback;
+      private DataDelegate _data_callback;
+      private SendConfirmationDelegate _on_send_onfirm_callback;
+      private readonly TimeSpan _connection_drop_threshold =
          new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
 
       public PubSubServerManagedTCPConnection(string serverURL)
          : this(new BlutoSubAddress(serverURL)) { }
 
-      public PubSubServerManagedTCPConnection(BlutoSubAddress serverAddress)
-      {
-         myServerAddress = serverAddress;
-         ClientToServerManagedTCPConnector.connectTo(serverAddress, this);
+      public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
+         _server_address = server_address;
+         _connect_to_ps_server();
+      }
+
+      private void _connect_to_ps_server() {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
+         ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
       }
 
       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string confId)
-      {
-         if (myConnection != null)
-         {
-            myConnection.sendWithConfirmation(data, ref confId, new SendConfirmationDelegate(myOnSendConfirm));
-         }
-         else
-         {
-            confId = Guid.NewGuid().ToString();
-            myConfs.enqueue(new Pair<string, bool>(confId, false));
+      public void sendWithConfirmation(byte[] data, ref string conf_id) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
+         if (_connection != null) {
+            _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
+               if (_on_send_onfirm_callback != null)
+                  _on_send_onfirm_callback(msg_id, confirmed);
+            });
+         } else {
+            bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
+            conf_id = Guid.NewGuid().ToString();
+            _negative_unsent_confs.enqueue(conf_id);
          }
       }
 
-      public void send(byte[] data)
-      {
-         Console.WriteLine($"PubSubServerManagedTCPConnection.send server: {myServerAddress} data: {data.Length} myConnection : {myConnection != null}");
-         if (myConnection != null)
-         {
-            Console.WriteLine("PubSubServerManagedTCPConnection.send server: " + myServerAddress + "data: " + data.Length);
-            myConnection.send(data);
+      public void send(byte[] data) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
+         if (_connection != null) {
+            bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
+            _connection.send(data);
          }
       }
 
-      public void send(Stream str)
-      {
+      public void send(Stream str) {
          BinaryReader br = new BinaryReader(str);
          send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
       }
 
-      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
-      {
-         return myDataQueue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
+      public void send(PubSubMessage msg) {
+         send(msg.to_stream());
       }
 
-      public string serverURL
-      {
-         get { return myServerAddress.ToString(); }
+      public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
+         var bytes = msg.to_bytes();
+         sendWithConfirmation(bytes, ref conf_id);
       }
 
-      public string clientURL
-      {
-         get
-         {
+      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
+         return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
+      }
+
+      public string serverURL {
+         get { return _server_address.ToString(); }
+      }
+
+      public string clientURL {
+         get {
             string rv = "";
-            lock (this)
-            {
-               if (myConnection == null)
-               {
+            lock (this) {
+               if (_connection == null) {
                   rv = "";
-               }
-               else
-               {
-                  if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty)
-                  {
+               } else {
+                  if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) {
                      string ip;
                      int port;
-                     myConnection.getSelfAddress(out ip, out port);
+                     _connection.getSelfAddress(out ip, out port);
                      rv = new BlutoSubAddress(ip, port).ToString();
-                  }
-                  else
-                  {
+                  } else {
                      rv = "";
                   }
                }
@@ -116,42 +118,35 @@ namespace BCG
          }
       }
 
-      public void registerCallback(ConnectionStatusChange del)
-      {
-         myConnCallback = del;
+      public void registerCallback(ConnectionStatusChange del) {
+         _conn_callback = del;
       }
 
-      public void registerDataCallback(DataDelegate del)
-      {
-         myDataCallback = del;
+      public void registerDataCallback(DataDelegate del) {
+         _data_callback = del;
       }
 
-      public void registerSendConfirmationCallback(SendConfirmationDelegate del)
-      {
-         myOnSendConfirmCallback = del;
+      public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
+         _on_send_onfirm_callback = del;
       }
 
       #endregion
 
       #region IDisposable Members
 
-      private bool myIsDisposed = false;
-      public void Dispose()
-      {
-         Console.WriteLine("PubSubServerManagedTCPConnection.Dispose");
+      private bool _is_disposed = false;
+      public void Dispose() {
+         bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
          //throw new Exception("The method or operation is not implemented.");
-         if (myConnection != null)
-         {
-            (myConnection as AsyncTCPConnection).Dispose();
+         if (_connection != null) {
+            _connection.Dispose();
 
          }
-         myIsDisposed = true;
+         _is_disposed = true;
       }
-      public bool isDisposed
-      {
-         get
-         {
-            return myIsDisposed;
+      public bool isDisposed {
+         get {
+            return _is_disposed;
          }
 
       }
@@ -166,20 +161,42 @@ namespace BCG
       /// (long) is safe for multi threaded operations
       /// </summary>
       /// <param name="message"></param>
-      public void onData(RemoteMessage message)
-      {
-         Console.WriteLine($"PubSubServerManagedTCPConnection.onData msg:type: {message.messageType}");
-         if (message.messageType == RemoteMessageType.data)
-         {
-            if (myDataCallback != null)
-               myDataCallback(message.remoteConnectionInfo.ToString(), message.message);
-            else
-               myDataQueue.enqueue(message.message);
-         }
-         else
-         {
-            myLastHB = DateTime.Now;
+      public void onData(RemoteMessage message) {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
+         // TODO check for disconenction
+         switch (message.messageType) {
+            case RemoteMessageType.Data:
+               if (_data_callback != null)
+                  _data_callback(message.remoteConnectionInfo.ToString(), message.message);
+               else
+                  _in_data_queue.enqueue(message.message);
+               break;
+            case RemoteMessageType.Heartbeat:
+               _last_hb = DateTime.Now;
+               break;
+            case RemoteMessageType.Connection:
+               _conn_callback(_server_address.ToString(), true);
+               break;
+            case RemoteMessageType.Disconnection:
+               if (_conn_callback != null)
+                  _conn_callback(_server_address.ToString(), false);
+               _connection.Dispose();
+               _connect_to_ps_server();
+               break;
+
          }
+         // this stupid shit, caused duplicating og messages and 2 days of wild goose chase
+         // if (message.messageType == RemoteMessageType.Data)
+         // {
+         //    if (_data_callback != null)
+         //       _data_callback(message.remoteConnectionInfo.ToString(), message.message);
+         //    else
+         //       _in_data_queue.enqueue(message.message);
+         // }
+         // else
+         // {
+         //    _last_hb = DateTime.Now;
+         // }
       }
 
       #endregion
@@ -191,22 +208,52 @@ namespace BCG
       /// except onData)
       /// </summary>
       /// <param name="connection"></param>
-      public void onConnection(ITCPConnection connection)
-      {
+      public void onConnection(ITCPConnection connection) {
          string ip;
          int port;
          connection.getPeerAddress(out ip, out port);
          BlutoSubAddress add = new BlutoSubAddress(ip, port);
-         Console.WriteLine($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()}");
-         if (add.ToString() == myServerAddress.ToString())
-         {
-            myLastHB = DateTime.Now;
-            myTempNewConnection = connection;
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
+         if (add.ToString() == _server_address.ToString()) {
+            _last_hb = DateTime.Now;
+            _new_connection = connection;
          }
       }
 
       #endregion
 
+      void _dispose_and_reconnect_if_missing_hb() {
+         if (!_connection.isDisposed) {
+            TimeSpan hb_interval = DateTime.Now - _last_hb;
+            bool hb_missed = hb_interval > _connection_drop_threshold;
+            if (hb_missed) {
+               if (_conn_callback != null)
+                  _conn_callback(_server_address.ToString(), false);
+               bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
+               _connection.Dispose();
+               _connect_to_ps_server();
+            }
+         } else {
+            // already disposed, nothing to do
+         }
+      }
+
+      void _check_for_new_connection() {
+         if (_new_connection != null) {
+            _connection = _new_connection;
+            _new_connection = null; // since we are using its presence as a marker of nw connections
+            bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
+            _connection.register(this);
+            if (_conn_callback != null)
+               _conn_callback(_server_address.ToString(), true);
+            if (_on_send_onfirm_callback != null) {
+               var msg_ids = _negative_unsent_confs.dequeueAll();
+               foreach (var id in msg_ids)
+                  _on_send_onfirm_callback(id, false);
+            }
+         }
+      }
+
       #region IWorker Members
 
       /// <summary>
@@ -214,57 +261,57 @@ namespace BCG
       /// the design only allows for onConnection to be on a different
       /// thread.
       /// </summary>
-      public void doWork()
-      {
+      public void doWork() {
+         //_dispose_and_reconnect_if_missing_hb();
+         _check_for_new_connection();
          ///check for heartbeats and let clients know in 
          ///case of missing hb (connection breaking)
-         TimeSpan curr_hb_interval = DateTime.Now - myLastHB;
-         bool hb_missed = curr_hb_interval > myConnectionDropThreshold;
-         bool conn_disposed = myConnection != null && myConnection.isDisposed;
-         if (conn_disposed || hb_missed)
-         {
-            if (hb_missed)
-            {
-               // since conn breakage was diagnosed logically, lets physically close the connection
-               myConnection.Dispose();
-               string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
-               Console.WriteLine(dispose_msg);
-            }
-            myConnection = null; // this is used as a marker for a dead connection
-            if (myConnCallback != null)
-               myConnCallback(myServerAddress.ToString(), false);
-            string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
-            $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {curr_hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
-            Console.WriteLine(reconn_msg);
-            ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
-         }
-         if (myTempNewConnection != null)
-         {
-            if (myConnection != null)
-            {
-               // TODOds
-               // log error and close it
-               // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
-               //                                                 + " before a new connection is established");
-            }
-            Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
-            myConnection = myTempNewConnection;
-            myTempNewConnection = null;
-            myConnection.register(this);
-            myConfs.dequeueAll();
-            if (myConnCallback != null)
-               myConnCallback(myServerAddress.ToString(), true);
-         }
-         foreach (Pair<string, bool> p in myConfs.dequeueAll())
-            if (myOnSendConfirmCallback != null)
-               myOnSendConfirmCallback(p.first, p.second);
+         // TimeSpan hb_interval = DateTime.Now - myLastHB;
+         // bool hb_missed = hb_interval > myConnectionDropThreshold;
+         // bool conn_disposed = myConnection != null || myConnection.isDisposed;
+         // if (conn_disposed || hb_missed)
+         // {
+         //    if (hb_missed)
+         //    {
+         //       // since conn breakage was diagnosed logically, lets physically close the connection
+         //       myConnection.Dispose();
+         //       string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
+         //       Console.WriteLine(dispose_msg);
+         //    }
+         //    myConnection = null; // this is used as a marker for a dead connection
+         //    if (myConnCallback != null)
+         //       myConnCallback(myServerAddress.ToString(), false);
+         //    string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
+         //    $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
+         //    ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
+         // }
+         // if (myTempNewConnection != null)
+         // {
+         //    if (myConnection != null)
+         //    {
+         //       // TODOds
+         //       // log error and close it
+         //       // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
+         //       //                                                 + " before a new connection is established");
+         //    }
+         //    Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
+         //    myConnection = myTempNewConnection;
+         //    myTempNewConnection = null;
+         //    myConnection.register(this);
+         //    myConfs.dequeueAll();
+         //    if (myConnCallback != null)
+         //       myConnCallback(myServerAddress.ToString(), true);
+         // }
+         // foreach (Pair<string, bool> p in myConfs.dequeueAll())
+         //    if (myOnSendConfirmCallback != null)
+         //       myOnSendConfirmCallback(p.first, p.second);
       }
 
       #endregion
 
-      private void myOnSendConfirm(string messageId, bool confirmed)
-      {
-         myConfs.enqueue(new Pair<string, bool>(messageId, confirmed));
-      }
+      // private void myOnSendConfirm(string messageId, bool confirmed) {
+      //    _negative_confs.enqueue(new Pair<string, bool>(messageId, confirmed));
+      // }
+
    }
 }

src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs

@@ -7,144 +7,116 @@ using System.Text;
 
 
 using BCG;
-//using BCG.Utilities;
+using System.Collections.Generic;
 
-namespace BCG
-{
-   public class MetaData : ICloneable
-   {
-      private Hashtable myHash = new Hashtable();
+namespace BCG {
+   // Thhis "wraps" around the following 4 PubSub meta objects, serializes it on the client and 
+   // deserialize it on the pubsub 
+   // PubMetaData
+   // SubMetaData
+   // TransactionMetaData
+   // UnSubMetaData
+   public class MetaData : ICloneable {
+      private IDictionary _name_value_map = new Dictionary<string, string>();
       int mySize = -1;
 
-      public MetaData()
-      {}
+      public MetaData() { }
 
-      public MetaData(object o)
-      {copyFrom(o);}
+      public MetaData(object o) { copyFrom(o); }
 
-      public MetaData(Stream s)
-      {
+      public MetaData(Stream s) {
          long initPos = s.Position;
          load(s);
          mySize = Convert.ToInt32(s.Position - initPos);
       }
 
-      public string this[string name]
-      {
-         get 
-         {return (string) myHash[name];}
-         set 
-         {
+      public string this[string name] {
+         get { return (string)_name_value_map[name]; }
+         set {
             if (value != null)
-               myHash[name] = value;
+               _name_value_map[name] = value;
          }
       }
 
-      public void add(string name, string [] values)
-      {
+      private void _add_string_array(string name, string[] values) {
          StringBuilder sb = new StringBuilder();
-         foreach (string s in values)
-         {
+         foreach (string s in values) {
             if (sb.Length != 0)
                sb.Append(" ");
             sb.Append(s);
          }
-         myHash[name] = sb.ToString();
+         _name_value_map[name] = sb.ToString();
       }
 
-      public int getAsInt(string name)
-      {return Int32.Parse(this[name]);}
+      private int getAsInt(string name) { return Int32.Parse(this[name]); }
 
-      public string[] getAsStringArray (string name)
-      {
+      private string[] _get_as_string_array(string name) {
          if (this[name] != "" && this[name] != null)
             return this[name].Split(' ');
          else
             return new string[0];
       }
 
-      public int size()
-      {
-         if (mySize < 0)
-         {
+      public int size() {
+         if (mySize < 0) {
             MemoryStream ms = new MemoryStream();
             this.save(ms);
             return mySize = (int)ms.Position;
-         }
-         else
+         } else
             return mySize;
       }
 
-      public bool isEqual (MetaData md)
-      {
-         if (md.myHash.Keys.Count != myHash.Keys.Count)
+      private bool isEqual(MetaData md) {
+         if (md._name_value_map.Keys.Count != _name_value_map.Keys.Count)
             return false;
 
-         foreach (string k in myHash.Keys)
-            if (!myHash[k].Equals(md.myHash[k]))
+         foreach (string k in _name_value_map.Keys)
+            if (!_name_value_map[k].Equals(md._name_value_map[k]))
                return false;
          return true;
       }
 
-      public void load(Stream s)
-      {
-         myHash = new Hashtable();
-         string [] state = 
+      // loads the content of the stream into this instance's map
+      private void load(Stream s) {
+         _name_value_map = new Hashtable();
+         string[] state =
             new BinaryReader(s).ReadString().Split('\n');
 
-         for (int i = 0; i < state.Length-1; i+= 2)
-            myHash[state[i]] = state[i+1];
+         for (int i = 0; i < state.Length - 1; i += 2)
+            _name_value_map[state[i]] = state[i + 1];
       }
 
-      public void save(Stream s)
-      {
+      // writes the content of this instance to stream s
+      public void save(Stream s) {
          new BinaryWriter(s).Write(this.ToString());
       }
 
-      public override string ToString()
-      {
+      public override string ToString() {
          StringBuilder sb = new StringBuilder();
-         foreach (string key in myHash.Keys) 
-            sb.AppendFormat("{0}\n{1}\n",key,myHash[key]);
+         foreach (string key in _name_value_map.Keys)
+            sb.AppendFormat("{0}\n{1}\n", key, _name_value_map[key]);
          return sb.ToString();
       }
 
-      public void copyFrom(object o)
-      {
+      // iterates through the fields and enters the name, values into _hash
+      public void copyFrom(object o) {
          this["op-code"] = o.GetType().Name;
-
-         
-
-         foreach (FieldInfo fi in o.GetType().GetFields())
-         {
-         
+         foreach (FieldInfo fi in o.GetType().GetFields()) {
             if (fi.GetValue(o) == null)
                continue;
-            
             else if (fi.GetValue(o) is String[])
-              add(fi.Name, (string[])fi.GetValue(o));
-            
+               _add_string_array(fi.Name, (string[])fi.GetValue(o));
             else
-               myHash[fi.Name] = fi.GetValue(o).ToString();
-            
+               _name_value_map[fi.Name] = fi.GetValue(o).ToString();
+
          }
       }
-
-      public void copyTo(object o)
-      {
-         
-         
-         foreach (FieldInfo fi in o.GetType().GetFields()) 
-         {
-         
+      // copies the name value map into object o
+      public void copyTo(object o) {
+         foreach (FieldInfo fi in o.GetType().GetFields()) {
             string value = this[fi.Name];
-            //this should only be here temporarily, while the new decompression changes are waiting to go into prod. 
-            if (value == null)
-            {
-               //BlueEventLog.logEvent("Error: Metadata missing field " + fi.Name, BlueEventLog2Level.MissingMetadata);
-            }
-            else
-            {
+            if (value == null) {
+            } else {
                if (fi.FieldType.Equals(typeof(string)))
                   fi.SetValue(o, value);
                else if (fi.FieldType.Equals(typeof(int)))
@@ -167,11 +139,10 @@ namespace BCG
          }
       }
 
-      public object Clone()
-      {
+      public object Clone() {
          MemoryStream ms = new MemoryStream();
          this.save(ms);
-         ms.Seek(0,SeekOrigin.Begin);
+         ms.Seek(0, SeekOrigin.Begin);
          return new MetaData(ms);
       }
    }

src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs

@@ -1,25 +1,25 @@
-using System;
+// using System;
 
-namespace BCG
-{
-   [Serializable]
-   public class RenewMetaData 
-   {
-      public string subscriber;
-      public int lease;
-      public bool c;
-      
-      public SubAddress getSubAddress()
-      {return SubAddress.decode(subscriber);}
+// namespace BCG
+// {
+//    [Serializable]
+//    public class RenewMetaData 
+//    {
+//       public string subscriber;
+//       public int lease;
+//       public bool c;
 
-      public RenewMetaData()
-      {}
+//       public SubAddress getSubAddress()
+//       {return SubAddress.decode(subscriber);}
 
-      public RenewMetaData(string s, 
-         int lease)
-      {
-         this.subscriber = s;
-         this.lease = lease;
-      }
-   }
-}
+//       public RenewMetaData()
+//       {}
+
+//       public RenewMetaData(string s, 
+//          int lease)
+//       {
+//          this.subscriber = s;
+//          this.lease = lease;
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs

@@ -3,10 +3,8 @@ using BCG;
 //using BCG.Utilities;
 using System.Text.RegularExpressions;
 
-namespace BCG
-{
-   public class PubSubRoute : IComparable<PubSubRoute>
-   {
+namespace BCG {
+   public class PubSubRoute : IComparable<PubSubRoute> {
       public string serverAddress;
       public string tagRegExp;
       //route names are used in intermediary code (i.e. while substations are being deployed)
@@ -16,8 +14,7 @@ namespace BCG
 
       public PubSubRoute() { }
 
-      public PubSubRoute(string theServerAddress, string theTagRegExp)
-      {
+      public PubSubRoute(string theServerAddress, string theTagRegExp) {
          serverAddress = theServerAddress;
          tagRegExp = theTagRegExp;
       }
@@ -28,41 +25,39 @@ namespace BCG
       //    // tagRegExp = cfg.tagsRegex;
       // }
 
-      public string[] getAllPubSubs()
-      {
+      public string[] getAllPubSubs() {
          return serverAddress.Split(';');
       }
 
-      public Regex getRegex()
-      {
+      public Regex getRegex() {
          return myRegex ?? new Regex(tagRegExp);
       }
 
       #region IComparable<PubSubRoute> Members
 
-      public int CompareTo(PubSubRoute other)
-      {
+      public int CompareTo(PubSubRoute other) {
          return serverAddress == other.serverAddress ? tagRegExp.CompareTo(other.tagRegExp)
                                                       : serverAddress.CompareTo(other.serverAddress);
       }
 
       #endregion
 
-      public override bool Equals(object obj)
-      {
+      public override bool Equals(object obj) {
          if (!(obj is PubSubRoute) || obj == null)
             return false;
-         else
-         {
+         else {
             PubSubRoute other = obj as PubSubRoute;
             return serverAddress.Equals(other.serverAddress) &&
                      tagRegExp.Equals(other.tagRegExp);
          }
       }
 
-      public override int GetHashCode()
-      {
+      public override int GetHashCode() {
          return base.GetHashCode();
       }
+
+      public static PubSubRoute get_default_route() {
+         return new PubSubRoute("bluto://127.0.0.0.1:1976", ".*");
+      }
    }
 }

src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs

@@ -7,26 +7,22 @@ using BCG;
 //using BCG.Utilities;
 //using bts;
 
-namespace BCG
-{
-   public class PubSubRouter : IPubSubRouter
-   {
+namespace BCG {
+   public class PubSubRouter : IPubSubRouter {
       private Regex[] myPubRouteRegExps;
       private Regex[] mySubRouteRegExps;
       private PubSubRoute[] myPubRoutes;
       private PubSubRoute[] mySubRoutes;
+      private readonly string _temp_only_pubsub = "bluto://127.0.0.1:1976";
 
 
       public PubSubRouter() : this(
          PubSub3Config.instance.getSubRouting(),
-         PubSub3Config.instance.getPubRouting())
-      { }
+         PubSub3Config.instance.getPubRouting()) { }
 
-      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes)
-      { }
+      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
 
-      public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes)
-      {
+      public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes) {
          mySubRouteRegExps = new Regex[subRoutes.Length];
          myPubRouteRegExps = new Regex[pubRoutes.Length];
 
@@ -39,8 +35,7 @@ namespace BCG
             myPubRouteRegExps[i] = new Regex(pubRoutes[i].tagRegExp);
       }
 
-      private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes)
-      {
+      private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes) {
          for (int i = 0; i < routes.Length; i++)
             if (regexes[i].IsMatch(forTag))
                return i;
@@ -50,49 +45,43 @@ namespace BCG
 
       #region IPubSubRouter methods
 
-      private string myPickAServer(string tag, string[] urls)
-      {
+      private string myPickAServer(string tag, string[] urls) {
          if (urls.Length == 1)
             return urls[0];
-         else
-         {
+         else {
             uint hc = (uint)tag.GetHashCode();
             return urls[hc % urls.Length];
          }
       }
 
-      public string getPubURLForTag(string tag)
-      {
-         var rv = "bluto://127.0.0.1:1976"; //myPickAServer(tag, myPubRoutes[myGetRouteIndex(tag, myPubRoutes, myPubRouteRegExps)].getAllPubSubs());
+      public string getPubURLForTag(string tag) {
+         var rv = _temp_only_pubsub;
          bts.Log.Gen.info($"PubSubRouter.getPubURLForTag tag: {tag} route: {rv}");
          return rv;
       }
 
-      public string getSubURLForTag(string tag)
-      {
-         var rv = "bluto://127.0.0.1:1976"; // myPickAServer(tag, mySubRoutes[myGetRouteIndex(tag, mySubRoutes, mySubRouteRegExps)].getAllPubSubs());
+      public string getSubURLForTag(string tag) {
+         var rv = _temp_only_pubsub; // myPickAServer(tag, mySubRoutes[myGetRouteIndex(tag, mySubRoutes, mySubRouteRegExps)].getAllPubSubs());
          bts.Log.Gen.info($"PubSubRouter.getSubURLForTag tag: {tag} route: {rv}");
          return rv;
       }
 
-      public List<string> getAllPubSubSubURLs()
-      {
-         List<string> theURLs = new List<string>();
-
-         foreach (PubSubRoute subRoute in mySubRoutes)
-            theURLs.AddRange(subRoute.getAllPubSubs());
-         return theURLs;
+      public List<string> getAllPubSubSubURLs() {
+         // List<string> theURLs = new List<string>();
 
+         // foreach (PubSubRoute subRoute in mySubRoutes)
+         //    theURLs.AddRange(subRoute.getAllPubSubs());
+         // return theURLs;
+         return new List<string>() { _temp_only_pubsub };
       }
 
-      public List<string> getAllPubSubPubURLs()
-      {
-         List<string> theURLs = new List<string>();
-
-         foreach (PubSubRoute pubRoute in myPubRoutes)
-            theURLs.AddRange(pubRoute.getAllPubSubs());
-         return theURLs;
+      public List<string> getAllPubSubPubURLs() {
+         // List<string> theURLs = new List<string>();
 
+         // foreach (PubSubRoute pubRoute in myPubRoutes)
+         //    theURLs.AddRange(pubRoute.getAllPubSubs());
+         // return theURLs;
+         return new List<string>() { _temp_only_pubsub };
       }
       public event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
       public event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
@@ -100,10 +89,8 @@ namespace BCG
       #endregion
 
       //this function is put here so that the events are used, in order to prevent build errors. 
-      private void myNeverCallThisFunction()
-      {
-         if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null)
-         {
+      private void myNeverCallThisFunction() {
+         if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) {
             int aVar = 2;
             aVar = aVar + 2;
          }

src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs

@@ -3,37 +3,33 @@ using System.Collections;
 using System.Reflection;
 using BCG.Utilities;
 using System.Collections.Generic;
+using System.Reactive.Subjects;
 
-namespace BCG
-{
-	public class TagType2HandlerMap
-	{
+namespace BCG {
+   public class TagType2HandlerMap {
       private static TagType2HandlerMap myServerHandlerMap = null,
                                         myPublisherStateMap = null;
-      private Dictionary<string, IPubSubServerTagHandler> myTagTypeToPubHandlerMap = new Dictionary<string, IPubSubServerTagHandler>();
+      private Dictionary<string, IPubSubServerTagHandler> myTagTypeToPubHandlerMap =
+         new Dictionary<string, IPubSubServerTagHandler>();
 
-      public TagType2HandlerMap()
-      {
-         if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null)
-         {
-            foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries)
-            {
-               try
-               {
+      public TagType2HandlerMap() {
+         if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null) {
+            foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries) {
+               try {
+                  var ass_loc = assembly_location;
+                  bts.Log.Gen.info($"TagType2HandlerMap entry: {entry} assembly_location: {ass_loc}");
                   Assembly assem = Assembly.LoadFrom(entry.assembly);
-                  Type t = assem.GetType(entry.handlerClass, true, false);
+                  Type t = assem.GetType(entry.handlerClass, true, true);
                   object obj = Activator.CreateInstance(t);
                   IPubSubServerTagHandler tagHandler = obj as IPubSubServerTagHandler;
+                  bts.Log.Gen.info($"TagType2HandlerMap successfully created obj: {obj} of type: {t} from assem: {assem}");
                   string[] tagTypes = tagHandler.supportedTagTypes;
-                  foreach (string tagType in tagTypes)
-                  {
+                  foreach (string tagType in tagTypes) {
+                     bts.Log.Gen.info($"TagType2HandlerMap adding tagType: {tagType} tagHandler: {tagHandler}");
                      myTagTypeToPubHandlerMap[tagType] = tagHandler;
                   }
-               }
-               catch (Exception ex)
-               {
-                  // BlueEventLog.logEvent(string.Format("error happened while loading type from the assembly ", entry.handlerClass, entry.assembly), BlueEventLevel.Warning);
-                  // BlueEventLog.logEvent(ex);
+               } catch (Exception ex) {
+                  bts.Log.Gen.error($"TagType2HandlerMap threw exception ex: {ex}");
                   if (ex is OutOfMemoryException)
                      throw ex;
                }
@@ -41,30 +37,41 @@ namespace BCG
          }
       }
 
-      private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType)
-      {
+      public static string assembly_location {
+         get {
+            // string codeBase = Assembly.GetExecutingAssembly().CodeBase;
+            // UriBuilder uri = new UriBuilder(codeBase);
+            // string path = Uri.UnescapeDataString(uri.Path);
+            // return Path.GetDirectoryName(path);
+            // Get the executing assembly
+            Assembly executingAssembly = Assembly.GetExecutingAssembly();
+
+            // Get the location (path) of the executing assembly
+            string assembly_location = executingAssembly.Location;
+
+            return assembly_location;
+         }
+      }
+
+      private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType) {
          IPubSubServerTagHandler rv;
          myTagTypeToPubHandlerMap.TryGetValue(tagType, out rv);
+         bts.Log.Gen.info($"TagType2HandlerMap.myGetHandlerFromTagType tag_type: {tagType} rv: {rv}");
          return rv;
       }
 
-      public static IPubSubServerTagHandler getHandler(string type, bool isServer)
-      {
-         lock (typeof(IPubSubServerTagHandler))
-         {
-            if (isServer)
-            {
+      public static IPubSubServerTagHandler getHandler(string type, bool isServer) {
+         lock (typeof(IPubSubServerTagHandler)) {
+            if (isServer) {
                if (myServerHandlerMap == null)
                   myServerHandlerMap = new TagType2HandlerMap();
                return myServerHandlerMap.myGetHandlerFromTagType(type);
-            }
-            else
-            {
+            } else {
                if (myPublisherStateMap == null)
                   myPublisherStateMap = new TagType2HandlerMap();
                return myPublisherStateMap.myGetHandlerFromTagType(type);
             }
          }
       }
-	}
+   }
 }

src/DataStreamSuite/PubSub/Client/TagTypeToAssembly.cs

@@ -1,17 +1,23 @@
 using System;
+using System.IO;
+using System.Reflection;
 
-namespace BCG
-{
+namespace BCG {
    [Serializable]
-   public class TagTypeToAssembly
-   {
+   public class TagTypeToAssembly {
       //[Serializable]
-      public class Entry 
-      {
-         public string handlerClass;
-         public string assembly;
+      public class Entry {
+         public string handlerClass = "BCG.PubSubServerTagHandler";
+
+         public string assembly = Path.Combine(
+            Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!,
+            "DataStreamCore.dll");
+
+         public override string ToString() {
+            return $"handlerClass: {handlerClass} assembly: {assembly}";
+         }
       }
 
-      public Entry[] entries;
+      public Entry[] entries = new Entry[] { new Entry() };
    }
 }

src/DataStreamSuite/PubSub/Config/PubSub3Config.cs

@@ -5,17 +5,13 @@ using System.Text.RegularExpressions;
 //using BCG.Utilities.BlueConfiguration;
 //using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    public class PubSub3Config //: ConfigurationRequest
    {
       private static PubSub3Config myInstance;
-      public static PubSub3Config instance
-      {
-         get
-         {
-            if (myInstance == null)
-            {
+      public static PubSub3Config instance {
+         get {
+            if (myInstance == null) {
                myInstance = new PubSub3Config();
                //myInstance = (PubSub3Config) ConfigurationRequester.request(typeof(PubSub3Config));
             }
@@ -43,7 +39,7 @@ namespace BCG
       private string[] myPS3_CrushableTags = new string[] { };
 
       //moved from PS2
-      private TagTypeToAssembly myTagTypeToAssemblyMap;
+      private TagTypeToAssembly myTagTypeToAssemblyMap = new TagTypeToAssembly();
       private string myPS3_ServerQueue;
 
       //substation stuff
@@ -83,145 +79,112 @@ namespace BCG
 
       // bluecomm transition
       //public ConfigKeys.BlueComm_PSTransition BlueComm_PSTransition { get; set; }
-      public PubSub3Config()
-      {
+      public PubSub3Config() {
          //BlueComm_PSTransition = new BCG.ConfigKeys.BlueComm_PSTransition(true);
       }
 
-      public bool isValid()
-      {
+      public bool isValid() {
          return true;
       }
 
-      public int PS3_retriesOnNoBufferSpace
-      {
+      public int PS3_retriesOnNoBufferSpace {
          get { return myPS3_retriesOnNoBufferSpace; }
          set { myPS3_retriesOnNoBufferSpace = value; }
       }
 
-      public int PS3_millisecondsBetweenNoBufferSpaceRetries
-      {
+      public int PS3_millisecondsBetweenNoBufferSpaceRetries {
          get { return myPS3_millisecondsBetweenNoBufferSpaceRetries; }
          set { myPS3_millisecondsBetweenNoBufferSpaceRetries = value; }
       }
 
-      public int PS3_millisecondsBetweenTimeSliceUpdates
-      {
+      public int PS3_millisecondsBetweenTimeSliceUpdates {
          get { return myPS3_millisecondsBetweenTimeSliceUpdates; }
          set { myPS3_millisecondsBetweenTimeSliceUpdates = value; }
       }
 
 
-      public bool BlueComm_mustDie
-      {
+      public bool BlueComm_mustDie {
          get { return myBlueComm_mustDie; }
          set { myBlueComm_mustDie = value; }
       }
 
-      public int PS3_HeartBeatInterval
-      {
+      public int PS3_HeartBeatInterval {
          get { return myPS3_HeartBeatInterval; }
          set { myPS3_HeartBeatInterval = value; }
       }
 
-      public int PS3_DoItAllSecondsBetweenLogs
-      {
+      public int PS3_DoItAllSecondsBetweenLogs {
          get { return myPS3_DoItAllSecondsBetweenLogs; }
          set { myPS3_DoItAllSecondsBetweenLogs = value; }
       }
 
-      public int PS3_MissingHeartBeatsForGC
-      {
+      public int PS3_MissingHeartBeatsForGC {
          get { return myPS3_MissingHeartBeatsBeforeGC; }
          set { myPS3_MissingHeartBeatsBeforeGC = value; }
       }
 
-      public bool PS3_UseManagedSockets
-      {
+      public bool PS3_UseManagedSockets {
          get { return myPS3_UseManagedSockets; }
          set { myPS3_UseManagedSockets = value; }
       }
 
-      public int PS3_StateUpdatePeriod
-      {
+      public int PS3_StateUpdatePeriod {
          get { return myPS3_StateUpdatePeriod; }
          set { myPS3_StateUpdatePeriod = value; }
       }
 
-      public int PS3_StatisticsUpdatePeriod
-      {
+      public int PS3_StatisticsUpdatePeriod {
          get { return myPS3_StatisticsUpdatePeriod; }
          set { myPS3_StatisticsUpdatePeriod = value; }
       }
 
-      public int PS3_DisposedConnectionCheckPeriod
-      {
+      public int PS3_DisposedConnectionCheckPeriod {
          get { return myPS3_DisposedConnectionCheckPeriod; }
          set { myPS3_DisposedConnectionCheckPeriod = value; }
       }
 
-      public int PS3_MessageProcessingWarningThreshold
-      {
+      public int PS3_MessageProcessingWarningThreshold {
          get { return myPS3_MessageProcessingWarningThreshold; }
          set { myPS3_MessageProcessingWarningThreshold = value; }
       }
 
-      public int PS3_millisecondsBetweenSubInfoUpdates
-      {
+      public int PS3_millisecondsBetweenSubInfoUpdates {
          get { return myPS3_millisecondsBetweenSubInfoUpdates; }
          set { myPS3_millisecondsBetweenSubInfoUpdates = value; }
       }
 
-      public double PS3_LatencyReportThresholdMS
-      {
+      public double PS3_LatencyReportThresholdMS {
          get { return myPS3_LatencyReportThresholdMS; }
          set { myPS3_LatencyReportThresholdMS = value; }
       }
 
-      public int PS3_ServerPort
-      {
+      public int PS3_ServerPort {
          get { return myPS3_ServerPort; }
          set { myPS3_ServerPort = value; }
       }
 
-      public PubSubRoute[] getSubRouting()
-      { return getRouting(); }
+      public PubSubRoute[] getSubRouting() { return getRouting(); }
 
-      public PubSubRoute[] getPubRouting()
-      { return getRouting(); }
+      public PubSubRoute[] getPubRouting() { return getRouting(); }
 
-      public PubSubRoute[] getRouting()
-      {
+      public PubSubRoute[] getRouting() {
          // TODO
-         return new PubSubRoute[] { };
+         return new PubSubRoute[] { PubSubRoute.get_default_route() };
          // return getRouting(SitesCore.getMySiteName());
       }
 
-      public PubSubRoute[] getRouting(string forSite)
-      {
+      public PubSubRoute[] getRouting(string forSite) {
          // TODO
-         return new PubSubRoute[] { };
-         // if (sites_layout != null) {
-         //    foreach (var s in sites_layout.sites)
-         //       if (s.name == forSite) {
-         //          List<PubSubRoute> result = new List<PubSubRoute>();
-         //          foreach (var cfgRoute in s.pubSub.routes)
-         //             result.Add(new PubSubRoute(cfgRoute));
-         //          return result.ToArray();
-         //       }
-         // }
-         //BlueEventLog2.instance().assert(false, "Could not find site {0} in config", forSite);
-         return null;
+         return getRouting();
+
       }
 
-      public List<string> getAllPubSubs()
-      {
+      public List<string> getAllPubSubs() {
          return new List<string>();
          //return getAllPubSubs(SitesCore.getMySiteName());
       }
 
-      public List<string> getAllPubSubs(string forSite)
-      {
+      public List<string> getAllPubSubs(string forSite) {
          List<string> result = new List<string>();
          foreach (PubSubRoute r in getRouting(forSite))
             result.AddRange(r.getAllPubSubs());
@@ -233,17 +196,14 @@ namespace BCG
 
       // public ConfigKeys.sites_siteName sites_siteName { get; set; }
 
-      public TagTypeToAssembly PS3_TagTypeToAssemblyMap
-      {
+      public TagTypeToAssembly PS3_TagTypeToAssemblyMap {
          get { return myTagTypeToAssemblyMap; }
          set { myTagTypeToAssemblyMap = value; }
       }
 
       //[Description("PUBSUB: The Message queue to look for pub sub messages")]
-      public string PS3_ServerQueue
-      {
-         get
-         {
+      public string PS3_ServerQueue {
+         get {
             if (myPS3_ServerQueue == null || myPS3_ServerQueue == "")
                return "PubSubQueue";
             else
@@ -252,14 +212,12 @@ namespace BCG
          set { myPS3_ServerQueue = value; }
       }
 
-      public bool PS3_InternallyPublishSubInfo
-      {
+      public bool PS3_InternallyPublishSubInfo {
          get { return myPS3_InternallyPublishSubInfo; }
          set { myPS3_InternallyPublishSubInfo = value; }
       }
 
-      public string[] PS3_CrushableTags
-      {
+      public string[] PS3_CrushableTags {
          get { return myPS3_CrushableTags; }
          set { myPS3_CrushableTags = value; }
       }
@@ -299,95 +257,80 @@ namespace BCG
 
       // }
 
-      public bool PS3_UseSubstations
-      {
+      public bool PS3_UseSubstations {
          get { return myPS3_UseSubstations; }
          set { myPS3_UseSubstations = value; }
       }
 
-      public PubSubPortRange PS3_PortRange
-      {
+      public PubSubPortRange PS3_PortRange {
          get { return myPS3_PortRange; }
          set { myPS3_PortRange = value; }
       }
 
-      public bool PS3_BindToRandomPort
-      {
+      public bool PS3_BindToRandomPort {
          get { return myPS3_BindToRandomPort; }
          set { myPS3_BindToRandomPort = value; }
 
       }
 
-      public bool PS3_IsSubstation
-      {
+      public bool PS3_IsSubstation {
          get { return myPS3_IsSubstation; }
          set { myPS3_IsSubstation = value; }
 
       }
 
-      public bool PS3_UseDynamicClients
-      {
+      public bool PS3_UseDynamicClients {
          get { return myPS3_UseDynamicClients; }
          set { myPS3_UseDynamicClients = value; }
       }
 
-      public bool PS3_UseManagedClientsInNativeCode
-      {
+      public bool PS3_UseManagedClientsInNativeCode {
          get { return myPS3_UseManagedClientsInNativeCode; }
          set { myPS3_UseManagedClientsInNativeCode = value; }
       }
 
-      public string PS3_CompressedTagsRegex
-      {
+      public string PS3_CompressedTagsRegex {
          get { return myPS3_CompressedTagsRegex; }
 
          set { myPS3_CompressedTagsRegex = value; }
       }
 
-      public string PS3_DebugTags
-      {
+      public string PS3_DebugTags {
          get { return myPS3_DebugTags; }
          set { myPS3_DebugTags = value; }
       }
 
-      public string PS3_TagsToReportCompressionRatio
-      {
+      public string PS3_TagsToReportCompressionRatio {
          get { return myPS3_TagsToReportCompressionRatio; }
          set { myPS3_TagsToReportCompressionRatio = value; }
       }
 
-      public bool PS3_DisableMSMQ
-      {
+      public bool PS3_DisableMSMQ {
          get { return myPS3_DisableMSMQ; }
          set { myPS3_DisableMSMQ = true; }
       }
 
-      public string PS3_IPTranslation
-      {
+      public string PS3_IPTranslation {
          get { return myPS3_IPTranslation; }
          set { myPS3_IPTranslation = value; }
       }
 
-      public long PS3_GCActivationThreshold
-      {
+      public long PS3_GCActivationThreshold {
          get { return myGCActivationThreshold; }
          set { myGCActivationThreshold = value; }
       }
 
-      public int PS3_WaitPeriodAfterGC
-      {
+      public int PS3_WaitPeriodAfterGC {
          get { return myWaitPeriodAfterGC; }
          set { myWaitPeriodAfterGC = value; }
       }
 
-      public bool PS3_UsePrepublishedServerState
-      {
+      public bool PS3_UsePrepublishedServerState {
          get { return myPS3_UsePrepublishedServerState; }
          set { myPS3_UsePrepublishedServerState = value; }
       }
 
-      public int PS3_MaxPacketSize
-      {
+      public int PS3_MaxPacketSize {
          get { return myPS3_MaxPacketSize; }
          set { myPS3_MaxPacketSize = value; }
       }
@@ -404,21 +347,17 @@ namespace BCG
       //    set { myRecovery_PubSubCopies = value; }
       // }
 
-      public bool isTagDebugTag(string tag)
-      {
-         if (myDebugTagsRegex == null)
-         {
+      public bool isTagDebugTag(string tag) {
+         if (myDebugTagsRegex == null) {
             myDebugTagsRegex = new Regex(myPS3_DebugTags, RegexOptions.IgnoreCase);
          }
          return myDebugTagsRegex.IsMatch(tag);
 
       }
 
-      public bool isTagCompressed(string tag)
-      {
+      public bool isTagCompressed(string tag) {
 
-         if (myCompressedTagsRegex == null)
-         {
+         if (myCompressedTagsRegex == null) {
             myCompressedTagsRegex = new Regex(myPS3_CompressedTagsRegex, RegexOptions.IgnoreCase);
 
          }
@@ -426,10 +365,8 @@ namespace BCG
          return (myCompressedTagsRegex.IsMatch(tag));
       }
 
-      public bool shouldReportTagsCompressionRatio(string tag)
-      {
-         if (myTagsToReportCompressionRatioRegex == null)
-         {
+      public bool shouldReportTagsCompressionRatio(string tag) {
+         if (myTagsToReportCompressionRatioRegex == null) {
             myTagsToReportCompressionRatioRegex = new Regex(myPS3_TagsToReportCompressionRatio, RegexOptions.IgnoreCase);
          }
          return (myTagsToReportCompressionRatioRegex.IsMatch(tag));

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs

@@ -45,8 +45,7 @@ namespace BCG
          public SendConfirmationDelegate onSendDel;
 
          public AsyncMessage(byte[] theBuffer, int theOffset)
-            : this(theBuffer, theOffset, String.Empty, null)
-         { }
+            : this(theBuffer, theOffset, String.Empty, null) { }
 
          public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel)
          {
@@ -445,7 +444,7 @@ namespace BCG
                               myDataReceived = true;
                               Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}");
                            }
-                           myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.data, myPeer));
+                           myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer));
 
                         }
                         else if (msgType == 1)
@@ -456,7 +455,7 @@ namespace BCG
                               Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}");
                            }
                            myLastHBReceivedTime = DateTime.Now;
-                           myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.heartbeat, myPeer));
+                           myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer));
                         }
                         myReadSize();
                      }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs

@@ -0,0 +1,296 @@
+using BCG;
+
+using Google.Protobuf;
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+using System.Xml.Serialization;
+using System.Reflection;
+using BCG.Utilities.General;
+using System.Net.WebSockets;
+
+namespace BCG
+{
+
+   public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable
+   {
+      //private Socket _socket;
+      // tied to the socket above
+      private NetworkStream _nw_stream;
+      private readonly ConnectionSide _side;
+      ITCPMessageHandler _message_handler;
+      private int _msg_id = 0;
+      RemoteConnectionInfo _peer_address;
+      RemoteConnectionInfo _self_address;
+      private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
+      private DateTime _last_hb_sent = DateTime.MinValue;
+      private DateTime _last_hb_received = DateTime.MaxValue;
+      private static byte _msg_type_data = 1;
+      private static byte _msg_type_hb = 0;
+      private bool _is_disposed = false;
+      SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
+
+      public class QueuedMessage
+      {
+         public string id;
+         public SendConfirmationDelegate del;
+         public byte[] data;
+
+         public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data)
+         {
+            id = id;
+            del = del;
+            data = data;
+         }
+
+         public QueuedMessage(byte[] data)
+         : this(String.Empty, null, data) { }
+
+      }
+
+      public enum ConnectionSide
+      {
+         Client,
+         Server
+      }
+
+      public AsyncTCPConnection2(Socket socket, ConnectionSide side)
+      {
+         //_socket = socket;
+         _nw_stream = new NetworkStream(socket);
+         _side = side;
+         IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
+         _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
+
+         var loc_ep = socket.LocalEndPoint as IPEndPoint;
+         _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
+      }
+
+      public void register(ITCPMessageHandler message_handler)
+      {
+         _message_handler = message_handler;
+         Task.Run(async () => _keep_reading());
+         Task.Run(async () => await _keep_sending());
+         //_keep_reading();
+         // if add a _keep_sending threadis added then we can maek send a non async
+         // methid making it much easier to integrate with the rest
+      }
+
+      // this thread pulls 
+      private async Task _keep_sending()
+      {
+         bts.Log.Gen.info($"TCPConnection._keep_sending starting");
+         while (!_is_disposed)
+         {
+            try
+            {
+               QueuedMessage m = _data_send_queue.dequeue(100);
+               if (m == null)
+               {
+                  continue;
+               }
+               await send_async(m.data);
+               if (String.IsNullOrEmpty(m.id))
+               {
+                  m.del(m.id, true);
+               }
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
+               await Task.Delay(100);
+               _handle_error(ex);
+            }
+         }
+      }
+
+      private async Task _keep_reading()
+      {
+         bts.Log.Gen.info($"TCPConnection._keep_reading starting");
+         while (!_is_disposed)
+         {
+            try
+            {
+               byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
+               bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
+               await _handle_msg(msg);
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
+               await Task.Delay(100);
+               _handle_error(ex);
+            }
+         }
+      }
+
+      private async Task _handle_msg(byte[] msg)
+      {
+         // read the first byte to read the msg type and deal with it accordingly
+         BinaryReader br = new BinaryReader(new MemoryStream(msg));
+         byte msg_type = br.ReadByte();
+         if (msg_type == _msg_type_data)
+         {
+            byte[] data = br.ReadBytes(msg.Length - 1);
+            RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
+            _message_handler.onData(m);
+         }
+         else
+         {
+            RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
+            _message_handler.onData(m);
+         }
+      }
+
+      // not very efficient
+      private static byte[] _prefix_msg_type(byte[] data)
+      {
+         MemoryStream ms = new MemoryStream();
+         BinaryWriter bw = new BinaryWriter(ms);
+         bw.Write(_msg_type_data);
+         bw.Write(data);
+         ms.Position = 0;
+         return ms.ToArray();
+      }
+
+      private async void _handle_error(Exception ex)
+      {
+         bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
+         try
+         {
+            var socket = _nw_stream.Socket;
+            socket.Close();
+            socket.Dispose();
+         }
+         catch (Exception ex2)
+         {
+            bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
+         }
+         _is_disposed = true;
+         RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
+         _message_handler.onData(rm);
+      }
+
+      public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
+
+      public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del)
+      {
+         try
+         {
+            conf_id = (++_msg_id).ToString();
+            await TCPSocketUtils.send_bytes_async(_nw_stream, data);
+            del(conf_id, true);
+         }
+         catch (Exception ex)
+         {
+            _handle_error(ex);
+         }
+      }
+
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
+      {
+         conf_id = (++_msg_id).ToString();
+         send(data);
+         del(conf_id, true);
+      }
+
+      public void send(byte[] data)
+      {
+         Task.Run(async () => await send_async(data));
+         bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
+         //_data_send_queue.enqueue(new QueuedMessage(data));
+      }
+
+      public async Task send_async(byte[] data)
+      {
+         // prefix 4 bytes for type (data vs hb)
+         MemoryStream str = new MemoryStream();
+         BinaryWriter bw = new BinaryWriter(str);
+         bw.Write(_msg_type_data);
+         bw.Write(data);
+         try
+         {
+            await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
+         }
+         catch (Exception ex)
+         {
+            bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
+            _handle_error(ex);
+         }
+      }
+
+      public void getPeerAddress(out string ip, out int port)
+      {
+         ip = _peer_address.ip;
+         port = _peer_address.port;
+         ///.Net has some problems getting the port, so using
+         ///native code for that
+         //sockaddr addr = new sockaddr();
+         //int namelen = 16;
+         //getsockname(mySock.Handle, ref addr, ref namelen);
+         //port = addr.sin_port;
+      }
+
+      public RemoteConnectionInfo get_peer_address()
+      {
+         string ip; int port;
+         getPeerAddress(out ip, out port);
+         return new RemoteConnectionInfo(ip, port);
+      }
+
+      public RemoteConnectionInfo get_self_address()
+      {
+         string ip; int port;
+         getSelfAddress(out ip, out port);
+         return new RemoteConnectionInfo(ip, port);
+      }
+
+
+      public void getSelfAddress(out string ip, out int port)
+      {
+         ip = _self_address.ip;
+         port = _self_address.port;
+      }
+
+      private async void _send_hb()
+      {
+         if (_last_hb_sent - DateTime.Now > _hb_period)
+         {
+            MemoryStream ms = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(ms);
+            bw.Write(_msg_type_hb);
+            ms.Position = 0;
+            byte[] hb_msg = ms.ToArray();
+            await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
+         }
+      }
+
+      public void doWork()
+      {
+         _send_hb();
+      }
+
+      public void Dispose()
+      {
+         _nw_stream.Close();
+         _nw_stream.Dispose();
+         _is_disposed = true;
+      }
+
+      public bool isDisposed
+      {
+         get
+         {
+            return _is_disposed;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs

@@ -4,21 +4,19 @@ using System.Text;
 using System.Net;
 using System.Net.Sockets;
 using System.Threading;
+using System.Threading.Tasks;
 //using BCG.Utilities;
 //using BCG.Utilities.General;
 
-namespace BCG
-{
-   internal class SocketConnectionArgs
-   {
+namespace BCG {
+   internal class SocketConnectionArgs {
       public EndPoint ep;
       public Socket sock;
       public ITCPConnectionHandler handler;
       public string serverURL;
 
       internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
-         string theServerURL)
-      {
+         string theServerURL) {
          ep = theEp;
          sock = theSock;
          handler = theHandler;
@@ -26,46 +24,46 @@ namespace BCG
       }
    }
 
-   public class ClientToServerManagedTCPConnector
-   {
+   public class ClientToServerManagedTCPConnector {
       private static Random myRnd = new Random(DateTime.Now.Millisecond);
       private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
       private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
                            myWaitTimeBeforeRetryingLowerLimit = 1000;
 
-      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler)
-      {
-         // TODOds
-         // BlueEventLog.logEvent(String.Format("Connection being attempted to server {0}", serverAddress.ToString()),
-         //                         PS3LogLevels.connection);
+      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
          Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
          Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
          EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
          sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
       }
 
-      private static void myOnConnect(IAsyncResult res)
-      {
+      public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+      }
+
+      public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         Task.Run(async () => {
+            await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+         });
+      }
+
+      private static void myOnConnect(IAsyncResult res) {
          Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
          SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
-         try
-         {
+         try {
             args.sock.EndConnect(res);
-         }
-         catch (SocketException sex)
-         {
+         } catch (SocketException sex) {
             string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
             //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
             //Console.WriteLine(errMsg);
-            if (args.handler.isDisposed)
-            {
+            if (args.handler.isDisposed) {
                string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
                //Console.WriteLine(doneMsg);
                //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
                return;
-            }
-            else
-            {
+            } else {
                string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
                //Console.WriteLine(retryMsg);
                //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);

src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs

@@ -2,15 +2,13 @@ using System;
 using System.Collections.Generic;
 using System.Text;
 
-namespace BCG
-{
+namespace BCG {
    public delegate void SendConfirmationDelegate(string messageId, bool confirmed);
    /// <summary>
    /// This is the very basic connection that sits right 
-   /// on top of the sockets (unmanaged, managed whatever).
+   /// on top of the sockets (used on both server and client)
    /// </summary>
-   public interface ITCPConnection : IDisposable
-   {
+   public interface ITCPConnection : IDisposable {
       /// <summary>
       /// The delegate is fired signalling if the data send was
       /// successful or not
@@ -18,7 +16,7 @@ namespace BCG
       /// <param name="data"></param>
       /// <param name="confId"></param>
       /// <param name="del"></param>
-      void sendWithConfirmation (byte[] data, ref string confId, SendConfirmationDelegate del);
+      void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
       /// <summary>
       /// sends data to the other side. theres no guaranatee that
       /// data is delievered.
@@ -54,6 +52,6 @@ namespace BCG
       /// hence this is an indicator of whether the connection is
       /// alive or dead
       /// </summary>
-      bool isDisposed { get;}
+      bool isDisposed { get; }
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs

@@ -2,46 +2,51 @@ using System;
 using System.Collections.Generic;
 using System.Text;
 using System.Net;
+using bts.utils;
 
-namespace BCG
-{
-   public class RemoteConnectionInfo
-   {
+namespace BCG {
+   public class RemoteConnectionInfo {
       private string myIp;
       private int myPort;
       private IPEndPoint myEndPoint;
 
-      public RemoteConnectionInfo(string ip, int port)
-      {
+      public RemoteConnectionInfo(string ip, int port) {
          myIp = ip;
          myPort = port;
       }
 
       public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint)
-         : this (ip, port)
-      {
+         : this(ip, port) {
          myEndPoint = endPoint;
       }
 
-      public string ip
-      {
+      public string ip {
          get { return myIp; }
       }
 
-      public int port
-      {
+      public int port {
          get { return myPort; }
       }
 
-      public IPEndPoint endPoint
-      {
+      public IPEndPoint endPoint {
          get { return myEndPoint; }
       }
 
-      public override string ToString()
-      {
+      public override string ToString() {
          return ip + ":" + port;
       }
-      
+
+      public override int GetHashCode() {
+         return ip.GetHashCode() ^ port.GetHashCode();
+      }
+
+      public override bool Equals(object? obj) {
+         if (obj == null) {
+            return false;
+         }
+         RemoteConnectionInfo other = obj as RemoteConnectionInfo;
+         return other != null && other.ip == ip && other.port == port;
+      }
+
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/RemoteMessage.cs

@@ -9,7 +9,7 @@ namespace BCG
    /// </summary>
    public enum RemoteMessageType
    {
-      heartbeat, data, connection, disconnection
+      Heartbeat, Data, Connection, Disconnection
    }
 
    /// <summary>
@@ -45,5 +45,11 @@ namespace BCG
          get { return myConnInfo; }
       }
 
+      override public string ToString()
+      {
+         Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
+         return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
+      }
+
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs

@@ -5,39 +5,30 @@ using System.Text;
 //using BCG.Utilities.General;
 //using BCG.Utilities;
 
-namespace BCG
-{
-   public class ServerToClientManagedTCPConnectionAcceptor
-   {
+namespace BCG {
+   public class ServerToClientManagedTCPConnectionAcceptor {
 
       private static Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor> myPortToSingleServerAcceptorMap = new Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor>();
       private static object mySyncObject = new object();
 
-      public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan)
-      {
+      public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
          SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
          int portNo = theAcceptor.activateWithRandomPort(handler, clientHBTimeSpan);
 
-         lock (mySyncObject)
-         {
+         lock (mySyncObject) {
             myPortToSingleServerAcceptorMap[portNo] = theAcceptor;
 
          }
-         return portNo;         
+         return portNo;
       }
 
-      public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan)
-      {
-         lock (mySyncObject)
-         {
-            if (myPortToSingleServerAcceptorMap.ContainsKey(port))
-            {
+      public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
+         lock (mySyncObject) {
+            if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
                string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
                // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
                return port;
-            }
-            else
-            {
+            } else {
                SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
                int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan);
                myPortToSingleServerAcceptorMap[assignedport] = theAcceptor;
@@ -46,6 +37,23 @@ namespace BCG
          } //end lock
       } //end activate code
 
+      public static int accept_connections(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
+         lock (mySyncObject) {
+            if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
+               string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
+               // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
+               return port;
+            } else {
+               SingleServerToClientManagedTCPConnectionAcceptor ps_acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+               int assignedport = ps_acceptor.accept_on_port(handler, "127.0.0.1", port, clientHBTimeSpan);
+               myPortToSingleServerAcceptorMap[assignedport] = ps_acceptor;
+               return assignedport;
+            }
+         } //end lock
+      } //end activate code
+
+
+
 
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs

@@ -4,13 +4,12 @@ using System.Text;
 using System.Threading;
 using System.Net;
 using System.Net.Sockets;
+using System.Threading.Tasks;
 
 //using BCG.Utilities;
 
-namespace BCG
-{               
-   public class SingleServerToClientManagedTCPConnectionAcceptor
-   {
+namespace BCG {
+   public class SingleServerToClientManagedTCPConnectionAcceptor {
       private TimeSpan myHBTimeSpan,
                               myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250);
       private ITCPConnectionHandler myHandler;
@@ -18,58 +17,50 @@ namespace BCG
       private int myPort;
 
 
-      public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan)
-      {
+      public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
          myHBTimeSpan = clientHBTimeSpan;
          myHandler = handler;
          myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
          int port = 0;
          bool madeItThrough = false;
-//         BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
+         //         BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
          Random randomizer = new Random();
          PubSubPortRange theRange = PubSub3Config.instance.PS3_PortRange;
 
-        
+
          //if the range of ports is 1000, then trying 5,000 times means 
          //your odds of missing an open port are less than 1%
 
-         int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;         
+         int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;
          int totalAttempts = 0;
-         while (!madeItThrough)
-         {
-            port = randomizer.Next(); 
+         while (!madeItThrough) {
+            port = randomizer.Next();
             port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort));
 
-            try
-            {
+            try {
                myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port));
                madeItThrough = true;
-            }
-            catch (Exception)
-            {
+            } catch (Exception) {
                totalAttempts++;
-               if (totalAttempts < maxAttempts){
+               if (totalAttempts < maxAttempts) {
                   //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection);
-               }
-               else
-               {
+               } else {
                   // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details.");
                }
             }
          }
          myPort = port;
          myAcceptSock.Listen(250);
-         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); 
+         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
 
          return myPort;
       }
 
-      public int activate (ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan)
-      {
+      public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
          myHBTimeSpan = clientHBTimeSpan;
          myHandler = handler;
          myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-         IPEndPoint endpoint = new IPEndPoint (IPAddress.Any, port);
+         IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port);
          myAcceptSock.Bind(endpoint);
          myAcceptSock.Listen(250);
          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
@@ -77,15 +68,18 @@ namespace BCG
          return myPort;  // not sure why we need this class variable myPort, but I am just keeping it up with the existing code
       }
 
-      private void myOnAccept(IAsyncResult res)
-      {
+      public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) {
+         Task.Run(async () => {
+            await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1));
+         });
+         return port;
+      }
+
+      private void myOnAccept(IAsyncResult res) {
          Socket newSock;
-         try
-         {
+         try {
             newSock = myAcceptSock.EndAccept(res);
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             // BlueEventLog.logEvent(ex);
             Thread.Sleep(myWaitTimeBeforeRetrying);
             myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
@@ -98,10 +92,10 @@ namespace BCG
          ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan);
          myHandler.onConnection(newConn);
          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
-         string ip;int port;
-         newConn.getPeerAddress (out ip, out port);
+         string ip; int port;
+         newConn.getPeerAddress(out ip, out port);
          string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort);
-//         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
+         //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
          //Console.WriteLine(connMsg);       
       }
    }

src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs

@@ -0,0 +1,113 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+using System.Xml.Serialization;
+using System.Reflection;
+using BCG.Utilities.General;
+using System.Net.WebSockets;
+
+namespace BCG
+{
+   // utility functions for TCP socket async communications. Does the following
+   // client to server connect
+   // server to client connect accepting
+   // send data
+   // receive data
+   public static class TCPSocketUtils
+   {
+      #region socket opening and connecting
+      // keeps accepting connections forever
+      public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler,
+                                                            TimeSpan wait_after_error)
+      {
+         IPAddress ipAddress = String.IsNullOrEmpty(ip) ? IPAddress.Parse(ip) : IPAddress.Any;
+         var ip_end_point = new IPEndPoint(ipAddress, port);
+         Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+         listener.Bind(ip_end_point);
+         listener.Listen(10);
+         Socket client_socket = null;
+         while (true)
+         {
+            try
+            {
+               // keeps on accepting connections
+               client_socket = await listener.AcceptAsync();
+               AsyncTCPConnection2 new_conn = new(client_socket, AsyncTCPConnection2.ConnectionSide.Server);
+               handler.onConnection(new_conn);
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
+               Task.Delay(wait_after_error);
+            }
+         }
+      }
+
+      // connects to the server and ona  successful connection, calls con__handler.on_connection_async with an ITCPConnection.
+      // In case of failures to connect it waits for a given amount of time before retrying 
+      public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
+                                                TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024)
+      {
+         Socket client_socket = null;
+         for (int i = 0; i < n_max_attempts; i++)
+         {
+            try
+            {
+               string log_conn = $"{ip}:{server_port}";
+               bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} attempt no: {i}");
+               IPAddress server_ip_address = IPAddress.Parse(ip);
+               client_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+               await client_socket.ConnectAsync(server_ip_address, server_port);
+               bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} successfully connected after {i} attempts");
+               ITCPConnection new_conn = new AsyncTCPConnection2(client_socket, AsyncTCPConnection2.ConnectionSide.Client);
+               conn_handler.onConnection(new_conn);
+               break;
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
+               Task.Delay(wait_after_failed_attempt);
+            }
+         }
+         return client_socket;
+      }
+      #endregion
+
+      #region data sending/receiving
+
+      // prefixes the size of the data and sends it as 2 sends, for size and data
+      // (can be done as one send perhaps)
+      public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff)
+      {
+         var data_sz = data_buff.Length;
+         byte[] sz_buff = new byte[4];
+         // aend the size first
+         BinaryWriter bw = new BinaryWriter(new MemoryStream(sz_buff));
+         bw.Write(data_sz);
+         await nw_stream.WriteAsync(sz_buff, 0, 4);
+         await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
+      }
+
+      // reads the bytes on a connected socket, by doing the opposite of the above,
+      // reads the size first and then the data
+      public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream)
+      {
+         byte[] sz_buffer = new byte[4];
+         //Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
+         int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length);
+         BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer));
+         int msg_sz = br.ReadInt32();
+         byte[] data_buffer = new byte[msg_sz];
+         bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length);
+         //Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
+         return data_buffer;
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/PubSub/PubSubMessage.cs

@@ -4,31 +4,39 @@ using System.IO;
 using BCG;
 using BCG.Utilities;
 
-namespace BCG
-{
-   // Tuple of 
-   public class PubSubMessage
-   {
-      public PubSubMessage (MetaData metaData, Stream body)
-      {this.metaData = metaData; this.body = body;}
+namespace BCG {
+   // PubItems are published to the pubsub wrapped in this class. It contains PubMetaData, wrapped in the MetaData 
+   // object and the actualy published payload in the body stream 
+   public class PubSubMessage {
+      public PubSubMessage(MetaData metaData, Stream body) { this.meta_data = metaData; this.body = body; }
 
-      public PubSubMessage (MetaData metaData)
-      {this.metaData = metaData;}
+      public PubSubMessage(PubMetaData pub_meta_data, Stream body)
+      : this(new MetaData(pub_meta_data), body) { }
 
-      public MetaData metaData;
+      public PubSubMessage(MetaData metaData) { this.meta_data = metaData; }
+
+      public MetaData meta_data;
       public Stream body = new MemoryStream(0);
 
-      public Stream toStream()
-      {
+      public Stream to_stream() {
          MemoryStream ms = new MemoryStream();
-         metaData.save(ms);
-         if (body.Length == 0 || body.Length == body.Position)
+         meta_data.save(ms);
+         if (body.Length == 0 || body.Length == body.Position) {
+            bts.Log.Gen.error("Error: Trying to publishing message with no content.");
             Console.WriteLine("Error: Trying to publishing message with no content.");
+         }
          StreamUtilityMethods.copyTo(new Stream[] { body }, ms);
-
          ms.Position = 0;
          return ms;
       }
 
+      public byte[] to_bytes() {
+         return StreamUtilityMethods.getBytesFromStream(to_stream());
+      }
+
+      public override string ToString() {
+         return $"PubSubMessage meta:data: {meta_data.ToString()} body.Posiiton: {body.Position} body.Length: {body.Length}";
+      }
+
    }
 }

src/DataStreamSuite/PubSub/PubSubUpdate.cs

@@ -2,22 +2,21 @@ using System;
 using System.Collections;
 using System.IO;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// this is the data returned from the
    /// pubsub server
    /// </summary>
-   public class PubSubUpdate
-   {
-      
-      public PubSubUpdate(string tag, string tagType, Stream body)
-      { this.tag = tag; this.body = body; this.tagType = tagType; }
+   public class PubSubUpdate {
+
+      public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
 
       public string tag;
       public Stream body;
       public string tagType;
 
-      
+      public override string ToString() {
+         return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
+      }
    };
 }

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -7,171 +7,67 @@ using System.Text.RegularExpressions;
 
 using BCG.Utilities;
 
-namespace BCG
-{
-   public class PublisherCore : IPublisherCore
-   {
-      //private static BlueCommManaged.IPubSub myBCPS;
-      
-      // TODO: what is this?
-      // public static void setPubSub(BlueCommManaged.IPubSub ps)
-      // {
-      //    lock(typeof(PublisherCore))
-      //    {
-      //       BlueEventLog2.instance().assert(myBCPS == null, 
-      //          "PublisherCore.setPubSub() should be called only once and before any use of PublisherCore");
-      //       myBCPS = ps;
-      //    }
-      // }
+namespace BCG {
+   public class PublisherCore : IPublisherCore {
 
       public static string pubSubBlueCommPrefix = "ps/";
       public static string pubSubBlueCommTransactionPrefix = "tgps/";
       public static string pubSubBlueCommTransactionComponentPrefix = "tcps/";
-      // public static BlueCommManaged.IPubSub getPubSub()
-      // {
-      //    lock (typeof(PublisherCore)) {
-      //       return myBCPS;
-      //    }
-      // }
 
-      enum Destination
-      {
+      enum Destination {
          None,
          BlueComm,
          PubSub,
          Both
       }
 
-      //private IPublisherCore myBlueCommPublisher = null;
-      private IPublisherCore myPubSubPublisher = null;
-      private Dictionary<string, Destination> myPubDestinations =
-         new Dictionary<string, Destination>();     
+      private IPublisherCore _publisher_core = null;
 
-      public PublisherCore()
-      {
-         myPubSubPublisher = new StaticPublisherCore();
-         //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
+      public PublisherCore() {
+         _publisher_core = new StaticPublisherCore();
       }
 
-      public PublisherCore(IPubSubRouter theRouter)
-      {
-         myPubSubPublisher = new StaticPublisherCore(theRouter);
-         //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
+      public PublisherCore(IPubSubRouter theRouter) {
+         _publisher_core = new StaticPublisherCore(theRouter);
       }
 
-      public PublisherCore(string URL)
-      {
-         myPubSubPublisher = new StaticPublisherCore(URL);
-         //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
+      public PublisherCore(string URL) {
+         _publisher_core = new StaticPublisherCore(URL);
       }
 
       #region IPublisherCore Methods
 
-      public void Dispose()
-      {
-         myPubSubPublisher.Dispose();
-         //myBlueCommPublisher.Dispose();
+      public void Dispose() {
+         _publisher_core.Dispose();
       }
-      public int getNumberOfWaitingMessages() { return myPubSubPublisher.getNumberOfWaitingMessages(); }
+      public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
 
-      private Destination myGetPublishDestionation (string tag)
-      {
-         return Destination.PubSub;
-         // lock (this)
-         // {
-         //    if (PubSub3Config.instance.BlueComm_mustDie)
-         //       return Destination.PubSub;
-         //    else
-         //    {
-         //       Destination d;
-         //       return Destination.PubSub;
-         //       if (myPubDestinations.TryGetValue(tag, out d))
-         //          return d;
-         //       else
-         //       {
-         //          foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInPubSub)
-         //             if (Regex.IsMatch(tag, regexp))
-         //             {
-         //                d = Destination.PubSub;
-         //                foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInPubSub)
-         //                   if (Regex.IsMatch(tag, regexp2))
-         //                   {
-         //                      d = Destination.None;
-         //                      break;
-         //                   }
-         //                break;
-         //             }
-
-         //          if (!tag.StartsWith("mon"))
-         //             foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInBlueComm)
-         //                if (Regex.IsMatch(tag, regexp))
-         //                {
-         //                   if (d == Destination.PubSub)
-         //                      d = Destination.Both;
-         //                   else
-         //                      d = Destination.BlueComm;
-         //                   foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInBlueComm)
-         //                      if (Regex.IsMatch(tag, regexp2))
-         //                      {
-         //                         if (d == Destination.Both)
-         //                            d = Destination.PubSub;
-         //                         else if (d == Destination.BlueComm)
-         //                            d = Destination.None;
-         //                         break;
-         //                      }
-         //                   break;
-         //                }
-
-         //          myPubDestinations[tag] = d;
-         //          return d;
-         //       }
-         //    }
-         // }
-      }
-
-      public void publish(string tag, Stream content)
-      {
+      public void publish(string tag, Stream content) {
          publish(tag, "", content);
       }
 
-      public void publish(string tagType, string tag, Stream content)
-      {
-         Destination d = myGetPublishDestionation(tag);
-         if (d == Destination.PubSub || d == Destination.Both)
-            myPubSubPublisher.publish(tagType, tag, content);
-         if (d == Destination.BlueComm || d == Destination.Both){}
-            //myBlueCommPublisher.publish(tagType, tag, content);
+      public void publish(string tagType, string tag, Stream content) {
+         bts.Log.Gen.info($"PublisherCore.publish tag: {tag} stream.Length: {content.Length} stream.Position: {content.Position}");
+         _publisher_core.publish(tagType, tag, content);
       }
 
-      public void beginTransaction()
-      {
-         myPubSubPublisher.beginTransaction();
-         //myBlueCommPublisher.beginTransaction();
+      public void beginTransaction() {
+         _publisher_core.beginTransaction();
       }
 
-      public void commitTransaction()
-      {
-         myPubSubPublisher.commitTransaction();
-         //myBlueCommPublisher.commitTransaction();
+      public void commitTransaction() {
+         _publisher_core.commitTransaction();
       }
 
-      public void abortTransaction()
-      {
-         myPubSubPublisher.abortTransaction();
-         //myBlueCommPublisher.abortTransaction();
+      public void abortTransaction() {
+         _publisher_core.abortTransaction();
       }
 
-      public void clearState(string tagToClear)
-      {
-         Destination d = myGetPublishDestionation(tagToClear);
-         if (d == Destination.PubSub || d == Destination.Both)
-            myPubSubPublisher.clearState(tagToClear);
-         if (d == Destination.BlueComm || d == Destination.Both){}
-            //myBlueCommPublisher.clearState(tagToClear);
+      public void clearState(string tagToClear) {
+         _publisher_core.clearState(tagToClear);
       }
 
-      public bool inTransaction
-      { get { return myPubSubPublisher.inTransaction; } }
+      public bool inTransaction { get { return _publisher_core.inTransaction; } }
 
       #endregion
 

src/DataStreamSuite/PubSub/PubsubDeserializationController.cs

@@ -3,27 +3,25 @@ using System.IO;
 using System.Collections;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Controls (ie creates dynamically) an IPubsubDeserializer based on the type name which is 
    /// part of the published data. Since its also a IPubsubDeserializer, all subsequent calls
    /// are passed though to the interface
    /// </summary>
-   public class PubsubDeserializationController : IPubsubDeserializer
-   {
+   public class PubsubDeserializationController : IPubsubDeserializer {
       private IPubsubDeserializer myDeserializer;
 
       public PubsubDeserializationController() { }
 
-      public void update(Stream stream)
-      {
+      public void update(Stream stream) {
+         bts.Log.Gen.info("PubsubDeserializationController.update stream: {stream.Position} / {stream.Length}");
          byte[] ops;
          Array rows;
          deserialize(new Stream[] { stream }, out ops, out rows);
       }
-      public void update(byte[] bytes)
-      {
+      public void update(byte[] bytes) {
+         bts.Log.Gen.info("PubsubDeserializationController.update bytes: {bytes.Length}");
          byte[] ops;
          Array rows;
          deserialize(new Stream[] { new MemoryStream(bytes, 0, bytes.Length, false, true) }, out ops, out rows);
@@ -31,39 +29,36 @@ namespace BCG
 
       #region IPubsubDeserializer Members
 
-      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         if (myDeserializer != null)
-         {
+      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
+
+         if (myDeserializer != null) {
             myDeserializer.deserialize(streams, out theOps, out theRows);
-         }
-         else
-         {
+            bts.Log.Gen.info($"PubsubDeserializationController.deserialize myDeserializer: {myDeserializer} ops: {theOps.Length} theRows: {theRows.Length}");
+         } else {
             Stream totalStream = UtilityMethods.getAggregatedStream(streams);
             long initialPos = totalStream.Position;
             BinaryReader br = new BinaryReader(totalStream);
             byte opCode = br.ReadByte();
-            if (opCode == StreamOpCode.create)
-            {
+            bts.Log.Gen.info($"PubsubDeserializationController.deserialize opCode: {opCode}");
+            if (opCode == StreamOpCode.create) {
                string deserName = br.ReadString();
-               if (deserName == "BCG.PubsubUniversalDeserializer") // so we can play old replay files which would not include the assembly name
-                  deserName = "BCG.PubsubUniversalDeserializer, DataStreamGateway";
-               myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
-                  as IPubsubDeserializer;
+               // TODOds
+               if (deserName == "BCG.PubsubUniversalDeserializer, DataStreamGateway") // so we can play old replay files which would not include the assembly name
+                  deserName = "BCG.PubsubUniversalDeserializer, DataStreamCore";
+               myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) as IPubsubDeserializer;
                totalStream.Position = initialPos;
                myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
-            }
-            else
-            {
-               // BlueEventLog.logEvent("Received a non create before a create. Bad op code was " + opCode.ToString(), BlueEventLevel.Warning);
+               bts.Log.Gen.info($"PubsubDeserializationController.deserialize myDeserializer: {myDeserializer} ops: {theOps.Length} theRows: {theRows.Length}");
+
+            } else {
+               bts.Log.Gen.error($"Received a non create before a create. Bad op code was : {opCode} ");
                theOps = new byte[] { };
                theRows = new object[] { };
             }
          }
       }
 
-      public byte[] getState()
-      {
+      public byte[] getState() {
          return myDeserializer == null ? new byte[] { } : myDeserializer.getState();
       }
 

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -9,17 +9,14 @@ using System.IO;
 using System.Diagnostics;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
 
    public class ManagedTCPMessageHub : IPubsubMessageHub,
                                        ITCPConnectionHandler,
-                                       ITCPMessageHandler
-   {
+                                       ITCPMessageHandler {
       private int myPort;
       private bool myIsDisposed = false;
-      public int port
-      {
+      public int port {
          get { return myPort; }
          //property is read only
 
@@ -66,8 +63,7 @@ namespace BCG
       private DateTime myLastHeartbeatDeadCollectionTime = DateTime.Now;
       private DateTime myLastDisposedDeadCollectionTime = DateTime.Now;
 
-      public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod)
-      {
+      public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod) {
          myPort = port;
          myHeartbeatSendInterval = hbSendInterval;
          mySubscriberDropThreshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hbSendInterval.TotalMilliseconds * missingHBsForGC));
@@ -81,24 +77,21 @@ namespace BCG
 
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(DataDelegate del)
-      {
+      public void registerDataCallback(DataDelegate del) {
          // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
          myDataCb = del;
       }
 
-      public void registerConnectionCallback(ConnectionStatusChange del)
-      {
+      public void registerConnectionCallback(ConnectionStatusChange del) {
          //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
          myConnCb = del;
       }
 
-      public void activate()
-      {
+      public void activate() {
          if (PubSub3Config.instance.PS3_BindToRandomPort || myPort == 0)
             myPort = ServerToClientManagedTCPConnectionAcceptor.activateWithRandomPort(this, myHeartbeatSendInterval);
          else
-            myPort = ServerToClientManagedTCPConnectionAcceptor.activate(this, myPort, myHeartbeatSendInterval);
+            myPort = ServerToClientManagedTCPConnectionAcceptor.accept_connections(this, myPort, myHeartbeatSendInterval);
 
          string msg = String.Format("TCP Component is ready, listening on port {0} for incoming connections", myPort);
          Console.WriteLine(msg);
@@ -108,19 +101,16 @@ namespace BCG
          doItAllThread.Start();
       }
 
-      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements)
-      {
+      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) {
          return myReceivedData.dequeueMultipleElements(timeout, maxElements);
       }
 
-      public bool send(string receiver, byte[] data)
-      {
+      public bool send(string receiver, byte[] data) {
          mySendData.enqueue(new Pair<string, byte[]>(receiver, data));
          return true;
       }
 
-      public bool send(string receiver, System.IO.Stream str)
-      {
+      public bool send(string receiver, System.IO.Stream str) {
          BinaryReader br = new BinaryReader(str);
          return send(receiver, br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
       }
@@ -133,17 +123,13 @@ namespace BCG
       /// Reports back connected and disconnected connections
       /// via the ConnectionStatucChange callback
       /// </summary>
-      public void doWork()
-      {
-         if (myConnCb != null)
-         {
-            foreach (string nc in myNewConnectionURLs.dequeueAll())
-            {
+      public void doWork() {
+         if (myConnCb != null) {
+            foreach (string nc in myNewConnectionURLs.dequeueAll()) {
                //BlueEventLog.logEvent(String.Format("doWork : New client {0} connected, making connection callback", nc), PS3LogLevels.connection);
                myConnCb(nc, true);
             }
-            foreach (string dc in myDeadConnectionURLs.dequeueAll())
-            {
+            foreach (string dc in myDeadConnectionURLs.dequeueAll()) {
                //BlueEventLog.logEvent(String.Format("doWork : Client {0} disconnected, making disconnection callback", dc), PS3LogLevels.disconnection);
                myConnCb(dc, false);
             }
@@ -160,8 +146,7 @@ namespace BCG
       /// stored in the hashtable and used for later communication)
       /// </summary>
       /// <param name="connection"></param>
-      public void onConnection(ITCPConnection connection)
-      {
+      public void onConnection(ITCPConnection connection) {
          myNewConnections.enqueue(connection);
       }
 
@@ -169,48 +154,38 @@ namespace BCG
 
       #region ITCPMessageHandler Members
 
-      public void onData(RemoteMessage message)
-      {
+      public void onData(RemoteMessage message) {
          Console.WriteLine("ManagedTCPMessgageHub.onData recv msg :" + message.remoteConnectionInfo.ToString());
          string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString();
-         if (message.messageType == RemoteMessageType.data)
-         {
+         if (message.messageType == RemoteMessageType.Data) {
             if (myDataCb != null)
                myDataCb(sender, message.message);
             else
                myReceivedData.enqueue(new Pair<string, byte[]>(sender, message.message));
-         }
-         else if (message.messageType == RemoteMessageType.heartbeat)
+         } else if (message.messageType == RemoteMessageType.Heartbeat)
             myReceivedHeartbeats.enqueue(sender);
-         else
-         {
+         else {
             //BlueEventLog.logEvent("Unknown message type from the client", BlueEventLevel.Error);
          }
       }
 
       #endregion
 
-      class TimmingInfo
-      {
-         public TimmingInfo()
-         {
+      class TimmingInfo {
+         public TimmingInfo() {
             myTimer = new Stopwatch();
             reset();
          }
 
-         public void start()
-         {
-            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0)
-            {
+         public void start() {
+            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) {
                myTimer.Reset();
                myTimer.Start();
             }
          }
 
-         public void stop()
-         {
-            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0)
-            {
+         public void stop() {
+            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) {
                myTimer.Stop();
                myMin = Math.Min(myMin, myTimer.ElapsedMilliseconds);
                myMax = Math.Max(myMax, myTimer.ElapsedMilliseconds);
@@ -219,16 +194,14 @@ namespace BCG
             }
          }
 
-         public void reset()
-         {
+         public void reset() {
             myMin = Double.MaxValue;
             myMax = Double.MinValue;
             myTotal = 0;
             myNMeasurements = 0;
          }
 
-         public override string ToString()
-         {
+         public override string ToString() {
             if (myNMeasurements > 0)
                return String.Format("min={0} max={1} avg={2} nMeasurements={3}", myMin, myMax, myTotal / myNMeasurements, myNMeasurements);
             else
@@ -242,8 +215,7 @@ namespace BCG
          private double myTotal;
       }
 
-      private void myDoItAll()
-      {
+      private void myDoItAll() {
          Dictionary<string, TimmingInfo> times = new Dictionary<string, TimmingInfo>();
          times.Add("myHandleNewConnections", new TimmingInfo());
          times.Add("myCollectDisposedConnections", new TimmingInfo());
@@ -252,12 +224,9 @@ namespace BCG
 
          DateTime lastTimmingLog = DateTime.Now;
 
-         while (true)
-         {
-            try
-            {
-               while (true)
-               {
+         while (true) {
+            try {
+               while (true) {
                   ///new connections
                   times["myHandleNewConnections"].start();
                   myHandleNewConnections();
@@ -279,16 +248,12 @@ namespace BCG
                   Pair<string, byte[]> p = mySendData.dequeue(myReadSendQueueTimeout);
 
                   times["send"].start();
-                  if (p != null)
-                  {
+                  if (p != null) {
                      ITCPConnection conn;
                      myConnections.TryGetValue(p.first, out conn);
-                     if (conn != null)
-                     {
+                     if (conn != null) {
                         conn.send(p.second);
-                     }
-                     else
-                     {
+                     } else {
                         string warningMsg = String.Format("sending data to {0} failed, this client might have " +
                            "disconnected, in which case it will be garbage collected within 60 secs", p.first);
                         //BlueEventLog.logEvent(warningMsg, PS3LogLevels.disconnection);
@@ -297,25 +262,21 @@ namespace BCG
                   times["send"].stop();
 
                   if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0 &&
-                     (DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs)
-                  {
+                     (DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs) {
                      lastTimmingLog = DateTime.Now;
 
                      StringBuilder b = new StringBuilder();
                      b.AppendFormat("myDoItAll stats:\n");
                      b.AppendFormat("queue size: {0}\n", mySendData.getCount());
 
-                     foreach (string timerName in times.Keys)
-                     {
+                     foreach (string timerName in times.Keys) {
                         b.AppendFormat("{0}: {1}\n", timerName, times[timerName]);
                         times[timerName].reset();
                      }
                      //BlueEventLog.logEvent(b.ToString(), PS3LogLevels.performance);
                   }
                }
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
                //BlueEventLog.logEvent(ex);
                if (ex is OutOfMemoryException)
                   throw ex;
@@ -323,19 +284,15 @@ namespace BCG
          }
       }
 
-      private void myCollectDisposedConnections()
-      {
-         if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod)
-         {
+      private void myCollectDisposedConnections() {
+         if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod) {
             myLastDisposedDeadCollectionTime = DateTime.Now;
             List<string> disposedURLs = new List<string>();
-            foreach (KeyValuePair<string, ITCPConnection> p in myConnections)
-            {
+            foreach (KeyValuePair<string, ITCPConnection> p in myConnections) {
                if (p.Value.isDisposed)
                   disposedURLs.Add(p.Key);
             }
-            foreach (string url in disposedURLs)
-            {
+            foreach (string url in disposedURLs) {
                string disposeMsg = String.Format("Collecting disposed connections : {0} is disposed by the connection itself, " +
                   "connection count before deleting this = {1}", url, myConnections.Count);
                Console.WriteLine(disposeMsg);
@@ -346,10 +303,9 @@ namespace BCG
          }
       }
 
-      private void myCollectMissingHeartbeatConnections()
-      {
-         if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold)
-         {
+      private void myCollectMissingHeartbeatConnections() {
+         return;  // disabling HBs for now to simplify things for the demo
+         if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) {
             myLastHeartbeatDeadCollectionTime = DateTime.Now;
             ///these are all the hbs received bw the last
             ///and current collectDeadConnection calls
@@ -365,12 +321,10 @@ namespace BCG
             foreach (string clientURL in myConnections.Keys)
                if (!indexedHbs.ContainsKey(clientURL))
                   deadClients.Add(clientURL);
-            foreach (string deadClient in deadClients)
-            {
-               string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
-                                                   "connection count before deleting this = {1}", deadClient, myConnections.Count);                                                   
-               //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection);
-               Console.WriteLine(disposeMsg);
+            foreach (string deadClient in deadClients) {
+               string dispose_msg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
+                                                   "connection count before deleting this = {1}", deadClient, myConnections.Count);
+               bts.Log.Gen.error(dispose_msg);
                myConnections[deadClient].Dispose();
                myConnections.Remove(deadClient);
                myDeadConnectionURLs.enqueue(deadClient);
@@ -378,11 +332,9 @@ namespace BCG
          }
       }
 
-      private void myHandleNewConnections()
-      {
+      private void myHandleNewConnections() {
          IList<ITCPConnection> newConns = myNewConnections.dequeueAll();
-         foreach (ITCPConnection conn in newConns)
-         {
+         foreach (ITCPConnection conn in newConns) {
             string ip;
             int port;
             conn.getPeerAddress(out ip, out port);
@@ -399,13 +351,11 @@ namespace BCG
       }
 
       #region IDisposable Memebers
-      public void Dispose()
-      {
+      public void Dispose() {
          myIsDisposed = true;
 
       }
-      public bool isDisposed
-      {
+      public bool isDisposed {
          get { return myIsDisposed; }
       }
       #endregion

src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs

@@ -4,23 +4,20 @@ using System.Text;
 using BCG.Utilities.General;
 using System.IO;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
    /// A subscriber can subscribe to it, and find out of a PubSub is alive or 
    /// not. Infact more so, if the subsriptions are working or not.
    /// </summary>
-   internal class SystemHeartbeatPublisher : IWorker
-   {
+   internal class SystemHeartbeatPublisher : IWorker {
       public static string tagType = "System_Heartbeat";
       public static string tagName = "System_Heartbeat";
-     
+
       private static SystemHeartbeatPublisher mySelf;
 
-      private SystemHeartbeatPublisher ()
-      {
-         
+      private SystemHeartbeatPublisher() {
+
          ///this is using a multithreaded MasterWorker since the 
          ///regular one is called within the MessageReactor, which
          ///waits 2500 msecs before calling doWork. This 2500 msecs
@@ -29,16 +26,14 @@ namespace BCG
          MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500));
       }
 
-      public static void init()
-      {
+      public static void init() {
          if (mySelf == null)
             mySelf = new SystemHeartbeatPublisher();
       }
 
       #region IWorker Members
 
-      public void doWork()
-      {
+      public void doWork() {
          PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString);
          MetaData md = new MetaData(pub);
          MemoryStream ms = new MemoryStream();

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -16,10 +16,8 @@ using System.Collections.Generic;
 using mbts;
 using mbts.spark;
 
-namespace BCG
-{
-   public class PubSubMessageReactor
-   {
+namespace BCG {
+   public class PubSubMessageReactor {
       private ServerMessageHub myMessageHub;
       private delegate void MyMsgHandler(string sender, Stream msg, MetaData m);
       private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>();
@@ -39,32 +37,24 @@ namespace BCG
       /* This delegate allows pubsub components to be notified when the identity of the pubsub is known*/
       public delegate void IdentityFoundDelegate(string identity, bool isRoot);
       public event IdentityFoundDelegate onIdentityFound;
-      public void fireIdentityFound(string identity, bool isRoot)
-      {
+      public void fireIdentityFound(string identity, bool isRoot) {
          if (onIdentityFound != null)
             onIdentityFound(identity, isRoot);
       }
 
-      private IPubSubGovernor myGovernor;
-
-
       public Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>();
 
       public PubSubMessageReactor() : this(false, false, string.Empty, string.Empty) { }
       public PubSubMessageReactor(string q) : this(false, false, string.Empty, q) { }
 
-      public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq)
-      {
+      public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq) {
          myOperationToHandler.Add(typeof(PubMetaData).Name, new MyMsgHandler(myHandlePub));
          myOperationToHandler.Add(typeof(SubMetaData).Name, new MyMsgHandler(myHandleSub));
          myOperationToHandler.Add(typeof(UnSubMetaData).Name, new MyMsgHandler(myHandleUnsub));
          myOperationToHandler.Add(typeof(TransactionMetaData).Name, new MyMsgHandler(myHandleTransaction));
 
-         // TODO, governor needs to be removed since its subststaion retired code
-         // if (PubSub3Config.instance.PS3_Components.governor != null)
-         //    myGovernor = PubSubComponentFactory.buildGovernor(this);
 
-         //BlueEventLog.logEvent("System Initialized", PS3LogLevels.info);
+         bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
 
          myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
          myMessageHub = ServerMessageHub.instance;
@@ -75,93 +65,49 @@ namespace BCG
 
       #region message handlers
 
-      public PubSubState state
-      {
+      public PubSubState state {
          get { return myState; }
          //this variable is read only
 
       }
 
-      private void myHandlePub(string sender, Stream msg, MetaData m)
-      {
-
+      // all publishes to the pubsub end here
+      private void myHandlePub(string sender, Stream msg_str, MetaData meta_data) {
+         bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub sender: {sender} metadata: {meta_data} str.pos: {msg_str.Position} str.len: {msg_str.Length}");
          mySingleTagTimeSliceTimer.start();
          ///the position in the stream is set to where the data starts with,
          ///since the meta data was already read in the myProcessMsg
          PubMetaData pub = new PubMetaData();
-         m.copyTo(pub);
+         meta_data.copyTo(pub);
          pub.connection = sender;
-         Console.WriteLine("PubSubMessageReactor.myHandlePub sender: " + sender + " tag: " + pub.tag);
-
-         //check to see if this publish took a while to get here
          pub.warnIfLatencyExceedsThreshold();
-
-         //check the governor to see if this publish is allowed, if we have a governor
-         PubSubGovernorResponse response = PubSubGovernorResponse.Accepted;
-         if (myGovernor != null)
-            response = myGovernor.requestToPublish(pub, msg);
-
-         if (response != PubSubGovernorResponse.Accepted)
-         {
-            // BlueEventLog2.instance().logEvent(string.Concat("PubSub Governor ",
-            //                                                 myGovernor.GetType().Name,
-            //                                                 " returned response ",
-            //                                                 response.ToString(),
-            //                                                 " publish on tag",
-            //                                                 pub.tag,
-            //                                                 "was ignored."), BlueEventLog2Level.Warning);
-            return;
-         }
-
-
-         //report the fact that we have a publish on this tag, if it's a debug tag
-         if (PubSub3Config.instance.isTagDebugTag(pub.tag))
-         {
-            // BlueEventLog.logEvent(string.Concat(GetType().Name,
-            //                                     " got publish on tag ",
-            //                                     pub.tag,
-            //                                     " from app ", pub.publishingApp,
-            //                                     " over connection ",
-            //                                     pub.connection, ".\n",
-            //                                     pub.isCompressed ? "Data was compressed.\n" : "Data was not compresed.\n",
-            //                                     "total size: ",
-            //                                     pub.size),
-
-            //                                     PS3LogLevels.debug);
-
-
-         }
+         var subs_for_tag = myState.getSubscriberConnectionsForTag(pub.tag);
+         bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub {subs_for_tag.Count} subs for tag: {pub.tag}");
 
          ///for all the subscribers, sends the messages over straightaway
-         foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag))
-         {
+         foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag)) {
             ///the position is adjusted to zero, so as to include
             ///the meta data as part of the message
-            msg.Position = 0;
-            myMessageHub.send(address, msg);
+            msg_str.Position = 0;
+            bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub sending tag: {pub.tag} to {address} len: {msg_str.Length - msg_str.Position}");
+            myMessageHub.send(address, msg_str);
          }
 
-
-
          //reset the position of the message so we can decompress it appropriately
-         msg.Position = 0;
-         msg.Position = m.size();
+         msg_str.Position = 0;
+         msg_str.Position = meta_data.size();
 
-         if (pub.isCompressed)
-         {
-            long mdLength = msg.Position;
-            long compressedSize = msg.Length - mdLength;
-            if (compressedSize == 0)
-            {
+         if (pub.isCompressed) {
+            long mdLength = msg_str.Position;
+            long compressedSize = msg_str.Length - mdLength;
+            if (compressedSize == 0) {
                string errorMsg = "Error: tag " + pub.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
-               //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
+               bts.Log.Gen.error(errorMsg);
                return;
             }
-            msg = StreamDecompressor.decompressStream(msg);
-            long decompressedSize = msg.Length - mdLength;
-
-            if (decompressedSize != pub.decompressedSize || compressedSize != pub.size)
-            {
+            msg_str = StreamDecompressor.decompressStream(msg_str);
+            long decompressedSize = msg_str.Length - mdLength;
+            if (decompressedSize != pub.decompressedSize || compressedSize != pub.size) {
                string compressionWarning = string.Format("Warning: Stream for tag {0} decompressed from {1} to {2} bytes, with {3} byes of metadata. Metadata expected {4} to {5}. Tag state not updated.",
                                                                                                                   pub.tag,
                                                                                                                   compressedSize,
@@ -169,43 +115,30 @@ namespace BCG
                                                                                                                   mdLength,
                                                                                                                   pub.size,
                                                                                                                   pub.decompressedSize);
-
-               //BlueEventLog.logEvent(compressionWarning, BlueEventLevel.Error);
+               bts.Log.Gen.warn(compressionWarning);
                return;
-
             }
-
          }
 
          //give the user a notification of the compression status of the tag has changed. 
          if ((!(myTagCompressionState.ContainsKey(pub.tag)) ||
-                myTagCompressionState[pub.tag] != pub.isCompressed))
-         {
+                myTagCompressionState[pub.tag] != pub.isCompressed)) {
             string compressionMessage;
             myTagCompressionState[pub.tag] = pub.isCompressed;
 
-            if (pub.isCompressed)
-            {
+            if (pub.isCompressed) {
                compressionMessage = "Tag " + pub.tag + " is compressed.";
                //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio);
-            }
-            else
-            {
+            } else {
                compressionMessage = "Tag " + pub.tag + " is not compressed.";
                //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio);
-
             }
-
          }
-
-
          ///updating state
-         msg.Position = 0;
-         msg.Position += m.size();
-         myState.updatePubData(pub, msg);
-
+         msg_str.Position = 0;
+         msg_str.Position += meta_data.size();
+         myState.updatePubData(pub, msg_str);
          mySingleTagTimeSliceTimer.stop();
-
          double thisTagProcessingTime = mySingleTagTimeSliceTimer.lastIntervalTime;
          if (onSingleTagPub != null)
             onSingleTagPub(pub.tag, thisTagProcessingTime);
@@ -213,8 +146,8 @@ namespace BCG
 
       }
 
-      private void myHandleSub(string sender, Stream msg, MetaData m)
-      {
+      private void myHandleSub(string sender, Stream msg, MetaData m) {
+         bts.Log.Gen.info($"PubSubMessageReactor.myHandleSub sender: {sender} metadata: {m} str.pos: {msg.Position} str.len: {msg.Length}");
          SubMetaData sub = new SubMetaData();
          m.copyTo(sub);
          sub.connection = sender;
@@ -222,12 +155,10 @@ namespace BCG
          ///update the state
          myState.sub(sub);
          ///to all these new subscriber s, send the existing data
-         foreach (string tag in sub.tags)
-         {
+         foreach (string tag in sub.tags) {
             byte[] data = myState.getContent(tag);
             //ie data has been published once
-            if (data != null)
-            {
+            if (data != null) {
                string tagType = myState.getTagType(tag);
                ///passing "" as the 4th arg, as PubMeta is used both by the
                ///publisher to publish data to the server (where the publisher
@@ -237,17 +168,14 @@ namespace BCG
                ///needed as of now, so its left black to save a few bytes
                PubMetaData pub;
                MemoryStream realContent;
-               if (PubSub3Config.instance.isTagCompressed(tag))
-               {
+               if (PubSub3Config.instance.isTagCompressed(tag)) {
                   SharpZipCompressor Kompessor = new SharpZipCompressor();
                   realContent = new MemoryStream();
                   Kompessor.compress(new MemoryStream(data), realContent);
 
                   pub = new PubMetaData(tagType, tag, (int)realContent.Length, (int)data.Length, "", "");
 
-               }
-               else
-               {
+               } else {
                   realContent = new MemoryStream(data);
                   pub = new PubMetaData(tagType, tag, data.Length, "", "");
                }
@@ -262,8 +190,7 @@ namespace BCG
          }
       }
 
-      private void myHandleUnsub(string sender, Stream msg, MetaData m)
-      {
+      private void myHandleUnsub(string sender, Stream msg, MetaData m) {
          UnSubMetaData unsub = new UnSubMetaData();
          m.copyTo(unsub);
          if (unsub.connection.Trim().Length == 0)
@@ -271,8 +198,7 @@ namespace BCG
          myState.unsub(unsub);
       }
 
-      private void myHandleTransaction(string sender, Stream msg, MetaData md)
-      {
+      private void myHandleTransaction(string sender, Stream msg, MetaData md) {
          mySingleTagTimeSliceTimer.start();
          TransactionMetaData t = new TransactionMetaData();
          md.copyTo(t);
@@ -280,8 +206,7 @@ namespace BCG
          ///form the structure tag->byte[]*
          Dictionary<string, byte[]> tagToMessages = new Dictionary<string, byte[]>();
          BinaryReader br = new BinaryReader(msg);
-         for (int i = 0; i < nMessages; i++)
-         {
+         for (int i = 0; i < nMessages; i++) {
             MetaData pubMd = new MetaData(msg);
             PubMetaData pub = new PubMetaData();
             pubMd.copyTo(pub);
@@ -289,8 +214,7 @@ namespace BCG
             byte[] tagData = new byte[pub.size];
             msg.Read(tagData, 0, tagData.Length);
             //report the fact that we have a publish on this tag, if it's a debug tag
-            if (PubSub3Config.instance.isTagDebugTag(pub.tag))
-            {
+            if (PubSub3Config.instance.isTagDebugTag(pub.tag)) {
                // BlueEventLog.logEvent(string.Concat(GetType().Name,
                //                                     " got publish on tag ",
                //                                     pub.tag,
@@ -315,13 +239,11 @@ namespace BCG
          }
          ///this gives subscriber->tag*
          Dictionary<string, List<string>> subscriberToTags = myState.getSubscriberConnectionsForTags(tagToMessages.Keys);
-         foreach (KeyValuePair<string, List<string>> kvp in subscriberToTags)
-         {
+         foreach (KeyValuePair<string, List<string>> kvp in subscriberToTags) {
             MemoryStream subStream = new MemoryStream();
             TransactionMetaData subTransaction = new TransactionMetaData(kvp.Value.Count);
             new MetaData(subTransaction).save(subStream);
-            foreach (string tag in kvp.Value)
-            {
+            foreach (string tag in kvp.Value) {
                byte[] tagData = tagToMessages[tag];
                PubMetaData pub = new PubMetaData(myState.getTagType(tag), tag, tagData.Length, "", "");
                new MetaData(pub).save(subStream);
@@ -336,8 +258,7 @@ namespace BCG
          double timeExpended = mySingleTagTimeSliceTimer.lastIntervalTime;
          //Don't try to divide by zero
          int numTags = tagToMessages.Keys.Count;
-         if (numTags != 0)
-         {
+         if (numTags != 0) {
             double timeExpendedPerTag = timeExpended / tagToMessages.Keys.Count;
             if (onSingleTagPub != null)
                foreach (string tag in tagToMessages.Keys)
@@ -351,18 +272,16 @@ namespace BCG
       /// </summary>
       /// <param name="msg"></param>
       /// <returns></returns>
-      private void myProcessMsg(string sender, Stream msg, out MetaData md)
-      {
-         md = new MetaData(msg);
+      private void myProcessMsg(string sender, Stream msg, out MetaData md) {
+         md = new MetaData(msg); // msg's Posiiton moved to end of meta data
          MyMsgHandler op = myOperationToHandler[md["op-code"]];
-         //BlueEventLog.logEvent(string.Format("process message {0}", m));
+         bts.Log.Gen.info($"PubSubMessageReactor.myProcessMsg sender: {sender} metadata: {md}");
          op(sender, msg, md);
       }
 
       #endregion
 
-      private void myHandleConnections(string client, bool connected)
-      {
+      private void myHandleConnections(string client, bool connected) {
          Console.WriteLine("PubSubMessageReactor.myHandleConnections client: " + client + " connected: " + connected);
          ///interested in disconnections only, as connections of no
          ///significance until they subscribe, in which case an entry
@@ -371,13 +290,11 @@ namespace BCG
             myState.removeConnection(client);
       }
 
-      public void stop()
-      {
+      public void stop() {
          myStop = true;
       }
 
-      public void start()
-      {
+      public void start() {
          myReceivingProcThread.Start();
       }
 
@@ -385,8 +302,7 @@ namespace BCG
       /// loops around and reads messages from the messageReceiver. calls processMsg
       /// passing in the IClientMessage. also logs stuff about the message.
       /// </summary>
-      private void myReceivingThreadProc()
-      {
+      private void myReceivingThreadProc() {
          TimeSpan receiveTimeout = new TimeSpan(0, 0, 0, 0, 2500);
          myMessageHub.activate();
 
@@ -398,31 +314,25 @@ namespace BCG
          string sender;
          Stream msg;
          MetaData meta;
-         while (!myStop)
-         {
+         while (!myStop) {
             sender = null;
             msg = null;
             meta = null;
-            try
-            {
+            try {
                List<Pair<string, byte[]>> receivedData = myMessageHub.getUpdates(receiveTimeout, 1);
-               if (receivedData.Count > 0)
-               {
+               if (receivedData.Count > 0) {
                   sender = receivedData[0].first;
                   msg = new MemoryStream(receivedData[0].second);
                   myTimer.start();
                   myProcessMsg(sender, msg, out meta);
                   myTimer.stop();
-                  if (myTimer.lastIntervalTime > myProcessingWarningThreshold)
-                  {
+                  if (myTimer.lastIntervalTime > myProcessingWarningThreshold) {
                      string warningMsg = String.Format("processing metadata {0} took {1} msecs", meta, myTimer.lastIntervalTime);
                      //BlueEventLog.logEvent(warningMsg, PS3LogLevels.performance);
                   }
                }
                myMasterWorker.doWork();
-            }
-            catch (Exception e)
-            {
+            } catch (Exception e) {
                //BlueEventLog.logEvent(e, BlueEventLevel.Error);
                Console.WriteLine("{0} : exception handling message: {1}\n", DateTime.Now, e.Message);
                string errMsg = String.Format("Following information about the meta data and sender is available : " +

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -7,21 +7,19 @@ using System.IO;
 
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Mantains all the state that there is in the server.
    /// This information is keyed off the tags (rather than 
    /// keyed off subscribers or publishers)
-   /// State information consists of 2 distints sets :
+   /// State information consists of 2 distinct sets :
    /// myTagToData consists of all the data published to 
    /// this pubsub.
    /// myTagToTagInfo consists of all the subscription and
    /// publishing information
    /// </summary>
    /// 
-   public class PubSubState
-   {
+   public class PubSubState {
 
       public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent);
       public event PubSubStateSingleTagChangeDelegate onSingleTagChange;
@@ -33,8 +31,7 @@ namespace BCG
       private Dictionary<string, SingleTagData> myTagToData = new Dictionary<string, SingleTagData>();
       private Dictionary<string, SingleTagPubSubMap> myTagToPubsubMap = new Dictionary<string, SingleTagPubSubMap>();
 
-      public PubSubState()
-      {
+      public PubSubState() {
          SystemSubscriptionMapPublisher.init(this);
          SystemStatsPublisher.init(this);
          SystemHeartbeatPublisher.init();
@@ -49,13 +46,10 @@ namespace BCG
       /// </summary>
       /// <param name="sub"></param>
       /// <param name="md"></param>
-      public void sub(SubMetaData sub)
-      {
+      public void sub(SubMetaData sub) {
          ///tag based key ds update
-         foreach (string tag in sub.tags)
-         {
-            if (!myTagToPubsubMap.ContainsKey(tag))
-            {
+         foreach (string tag in sub.tags) {
+            if (!myTagToPubsubMap.ContainsKey(tag)) {
                myTagToPubsubMap.Add(tag, new SingleTagPubSubMap(tag));
             }
             myTagToPubsubMap[tag].onSub(sub);
@@ -67,20 +61,15 @@ namespace BCG
             onTagMapChanged(sub.tags);
       }
 
-      public void unsub(UnSubMetaData unsub)
-      {
-         foreach (string tag in unsub.tags)
-         {
-            if (myTagToPubsubMap.ContainsKey(tag))
-            {
+      public void unsub(UnSubMetaData unsub) {
+         foreach (string tag in unsub.tags) {
+            if (myTagToPubsubMap.ContainsKey(tag)) {
                ///maybe check here, that if there are no subscribers
                ///and publishers for the tag then remove it.
                myTagToPubsubMap[tag].onUnsub(unsub);
                if (onSingleTagChange != null)
                   onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
-            }
-            else
-            {
+            } else {
                string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
                   " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
                //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
@@ -90,11 +79,9 @@ namespace BCG
             onTagMapChanged(unsub.tags);
       }
 
-      public void removeConnection(string connection)
-      {
+      public void removeConnection(string connection) {
          List<string> tagsEffected = new List<string>();
-         foreach (KeyValuePair<string, SingleTagPubSubMap> p in myTagToPubsubMap)
-         {
+         foreach (KeyValuePair<string, SingleTagPubSubMap> p in myTagToPubsubMap) {
             ///maybe check here, that if there are no subscribers
             ///and publishers for the tag then remove it.
             bool wasPublisher = false;
@@ -103,8 +90,7 @@ namespace BCG
             if (p.Value.onConnectionRemoveIfExists(connection, out wasPublisher, out wasSubscriber))
                tagsEffected.Add(tag);
 
-            if (onSingleTagChange != null)
-            {
+            if (onSingleTagChange != null) {
                //we don't know the app or key qualifier or anything for these connection. that is unfortunate. 
                if (wasPublisher)
                   onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, ""));
@@ -128,67 +114,60 @@ namespace BCG
       /// </summary>
       /// <param name="tag"></param>
       /// <param name="msg"></param>
-      public void updatePubData(PubMetaData pub, Stream msg)
-      {
+      public void updatePubData(PubMetaData pub, Stream msg) {
          int sz = (int)(msg.Length - msg.Position);
+         bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
          ///this is the only place where SingleTagData should be created
-         if (!myTagToData.ContainsKey(pub.tag))
-         {
+         if (!myTagToData.ContainsKey(pub.tag)) {
             ///first time data is published for this tag
             myTagToData.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
          }
          myTagToData[pub.tag].pubUpdate(msg);
 
-         if (!myTagToPubsubMap.ContainsKey(pub.tag))
-         {
+         if (!myTagToPubsubMap.ContainsKey(pub.tag)) {
             myTagToPubsubMap.Add(pub.tag, new SingleTagPubSubMap(pub.tag));
          }
          bool pubChanged = false;
          myTagToPubsubMap[pub.tag].onPub(pub, out pubChanged);
-         if (pubChanged)
-         { 
+         if (pubChanged) {
             if (onTagMapChanged != null)
                onTagMapChanged(new string[] { pub.tag });
 
             if (onSingleTagChange != null)
                onSingleTagChange(
-                  new PubSubSingleTagStateEvent(pub.tag, 
-                     PubSubSingleTagStateEvent.EventType.publish, 
+                  new PubSubSingleTagStateEvent(pub.tag,
+                     PubSubSingleTagStateEvent.EventType.publish,
                      pub.connection, pub.publishingApp));
          }
          if (onPublishStats != null)
             onPublishStats(pub.tag, sz);
-         
+
       }
 
       #endregion
 
-      public ICollection<string> getSubscriberConnectionsForTag(string tag)
-      {
+      public ICollection<string> getSubscriberConnectionsForTag(string tag) {
          if (myTagToPubsubMap.ContainsKey(tag))
             return myTagToPubsubMap[tag].subscriberConnections;
          else
             return new string[] { };
       }
 
-      public ICollection<string> getSubscribingAppsForTag(string tag)
-      {
+      public ICollection<string> getSubscribingAppsForTag(string tag) {
          if (myTagToPubsubMap.ContainsKey(tag))
             return myTagToPubsubMap[tag].subscripingApps;
          else
             return new string[] { };
       }
 
-      public byte[] getContent(string tag)
-      {
+      public byte[] getContent(string tag) {
          if (!myTagToData.ContainsKey(tag))
             return null;
          else
             return myTagToData[tag].getCurrentContent();
       }
 
-      public string getTagType(string tag)
-      {
+      public string getTagType(string tag) {
          if (!myTagToData.ContainsKey(tag))
             return null;
          else
@@ -207,29 +186,23 @@ namespace BCG
       /// <param name="deadPublishers"></param>
       public void getAllConnections(string tag, out Dictionary<string, string> subscribers,
                                                 out Dictionary<string, string> activePublishers,
-                                                out ICollection<Pair<string, string>> deadPublishers)
-      {
-         if (myTagToPubsubMap.ContainsKey(tag))
-         {
+                                                out ICollection<Pair<string, string>> deadPublishers) {
+         if (myTagToPubsubMap.ContainsKey(tag)) {
             SingleTagPubSubMap map = myTagToPubsubMap[tag];
             subscribers = map.subscribers;
             activePublishers = map.activePublishers;
             deadPublishers = map.deadPublishers;
-         }
-         else
-         {
+         } else {
             subscribers = activePublishers = new Dictionary<string, string>();
             deadPublishers = new List<Pair<string, string>>();
          }
       }
 
-      public ICollection<string> allTags
-      {
+      public ICollection<string> allTags {
          get { return myTagToPubsubMap.Keys; }
       }
 
-      public ICollection<string> tagsWithData
-      {
+      public ICollection<string> tagsWithData {
          get { return myTagToData.Keys; }
       }
 
@@ -239,15 +212,11 @@ namespace BCG
       /// </summary>
       /// <param name="tags"></param>
       /// <returns></returns>
-      public Dictionary<string, List<string>> getSubscriberConnectionsForTags(IEnumerable<string> tags)
-      {
+      public Dictionary<string, List<string>> getSubscriberConnectionsForTags(IEnumerable<string> tags) {
          Dictionary<string, List<string>> retVal = new Dictionary<string, List<string>>();
-         foreach (string tag in tags)
-         {
-            if (myTagToPubsubMap.ContainsKey(tag))
-            {
-               foreach (string sub in myTagToPubsubMap[tag].subscriberConnections)
-               {
+         foreach (string tag in tags) {
+            if (myTagToPubsubMap.ContainsKey(tag)) {
+               foreach (string sub in myTagToPubsubMap[tag].subscriberConnections) {
                   if (!retVal.ContainsKey(sub))
                      retVal.Add(sub, new List<string>());
                   retVal[sub].Add(tag);

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -7,21 +7,18 @@ using System.Collections.Generic;
 using BCG.Utilities;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SingleTagData consists of (as the name implies)
    /// the data published for a tag
    /// </summary>
-   internal class SingleTagData
-   {
+   internal class SingleTagData {
       private string myTagType = null;
       private string myTag;
       private byte[] myLastPublishedMsg;
       private IPubSubServerTagHandler myHandler;
 
-      public SingleTagData(string tag, string tagType)
-      {
+      public SingleTagData(string tag, string tagType) {
          myTag = tag;
          myTagType = tagType;
          if (myTagType == SystemSubscriptionMapPublisher.tagType)
@@ -29,6 +26,7 @@ namespace BCG
          else if (myTagType == SystemStatsPublisher.tagType)
             myHandler = SystemStatsPublisher.getInstance();
          else
+            // this is where it gets the stream handler
             myHandler = TagType2HandlerMap.getHandler(myTagType, true);
          if (myHandler != null)
             myHandler.register(new InternalPublishDelegate(myHandleHandlerCallbacks));
@@ -38,16 +36,13 @@ namespace BCG
 
       public string tagType { get { return myTagType; } }
 
-      internal void pubUpdate(Stream stream)
-      {
-         if (myHandler != null)
-         {
-            try
-            {
+      internal void pubUpdate(Stream stream) {
+         string handler_str = myHandler != null ? myHandler.GetType().ToString() : "NULL";
+         bts.Log.Gen.info($"SingleTagData.pubUpdate tag: {myTag} handler: {handler_str} stream: {stream.Position} / {stream.Length}");
+         if (myHandler != null) {
+            try {
                myHandler.update(myTag, new Stream[] { stream });
-            }
-            catch (Exception e)
-            {
+            } catch (Exception e) {
                // BlueEventLog.logEvent(e);
                string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + myTagType +
                                  " tag = " + myTag + " length = " + stream.Length + "\n" +
@@ -58,30 +53,22 @@ namespace BCG
                else
                   throw new ApplicationException(errMsg, e);
             }
-         }
-         else
-         {
+         } else {
             myLastPublishedMsg = new byte[stream.Length - stream.Position];
             stream.Read(myLastPublishedMsg, 0, myLastPublishedMsg.Length);
          }
       }
 
-      public byte[] getCurrentContent()
-      {
-         if (myHandler != null)
-         {
+      public byte[] getCurrentContent() {
+         if (myHandler != null) {
             return myHandler.getContent(myTag);
-         }
-         else
-         {
+         } else {
             return myLastPublishedMsg;
          }
       }
 
-      private void myHandleHandlerCallbacks(ICollection<Triplet<string, string, byte[]>> cbItems)
-      {
-         foreach (Triplet<string, string, byte[]> p in cbItems)
-         {
+      private void myHandleHandlerCallbacks(ICollection<Triplet<string, string, byte[]>> cbItems) {
+         foreach (Triplet<string, string, byte[]> p in cbItems) {
             string tagName = p.first, tagType = p.second;
             byte[] data = p.third;
             PubMetaData pub = new PubMetaData(tagType, tagName, data.Length, "local", ClientInfo.thisClient.processString);

src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs

@@ -7,8 +7,7 @@ using BCG.Utilities.General;
 using mbts;
 using mbts.spark;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Deals with the subscriptions (and getting data) from one
    /// pubsub server. Once again, theres no sharing of conenctions
@@ -17,15 +16,13 @@ namespace BCG
    /// its concerned with (passed as a conctructor arg).
    /// </summary>
 
-   internal interface ISingleServerSubscriberCore : IDisposable
-   {
+   internal interface ISingleServerSubscriberCore : IDisposable {
       IList<PubSubUpdate> getUpdates(int timeout);
       event PubSubConnectionChange onPubSubConnectionChange;
       void subscribe(params string[] tags);
       void unsubscribe(params string[] tags);
    }
-   internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore
-   {
+   internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
       private HighPerformanceTimer myTimer = new HighPerformanceTimer();
       private PubSubServerConnectionFactory myConnectionFactory;
       private IPubSubServerConnection myConnection;
@@ -33,15 +30,13 @@ namespace BCG
 
       public event PubSubConnectionChange onPubSubConnectionChange;
 
-      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory)
-      {
+      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
          myConnectionFactory = connectionFactory;
          myConnection = myConnectionFactory.getConnection(serverURL);
          myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
       }
 
-      public IList<PubSubUpdate> getUpdates()
-      {
+      public IList<PubSubUpdate> getUpdates() {
          return getUpdates(1);
       }
 
@@ -51,54 +46,41 @@ namespace BCG
       /// </summary>
       /// <param name="timeout"></param>
       /// <returns></returns>
-      public IList<PubSubUpdate> getUpdates(int timeout)
-      {
+      public IList<PubSubUpdate> getUpdates(int timeout) {
          myConnection.doWork();
          List<PubSubUpdate> retVal = new List<PubSubUpdate>();
          IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
-         foreach (byte[] data in dataItems)
-         {
+         foreach (byte[] data in dataItems) {
             processMessage(data, retVal);
          }
          return retVal;
-         
+
       }
 
-      public List<string> getSubscribedTags()
-      {
+      public List<string> getSubscribedTags() {
          return myTags;
       }
 
-      public void subscribe(params string[] tags)
-      {
-         foreach (string tag in tags)
-         {
-            if (myTags.Contains(tag))
-            {
+      public void subscribe(params string[] tags) {
+         foreach (string tag in tags) {
+            if (myTags.Contains(tag)) {
                string errMsg = String.Format("tag {0} was already subscribed from server {1}",
                   tag, myConnection.serverURL);
                // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
-            }
-            else
-            {
+            } else {
                myTags.Add(tag);
             }
          }
          mySubscribe(tags);
       }
 
-      public void unsubscribe(params string[] tags)
-      {
+      public void unsubscribe(params string[] tags) {
          List<string> unsubTags = new List<string>();
-         foreach (string tag in tags)
-         {
-            if (myTags.Contains(tag))
-            {
+         foreach (string tag in tags) {
+            if (myTags.Contains(tag)) {
                myTags.Remove(tag);
                unsubTags.Add(tag);
-            }
-            else
-            {
+            } else {
                string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
                   "it was never subscribed", tag, myConnection.serverURL);
                // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
@@ -108,13 +90,13 @@ namespace BCG
          Stream str = new MemoryStream();
          new MetaData(unsub).save(str);
          str.Position = 0;
+         // this is where data is actually sent out
          myConnection.send(str);
       }
 
       #region IDisposable Members
 
-      public void Dispose()
-      {
+      public void Dispose() {
          ///remove the subscriber
          //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
          myConnection.Dispose();
@@ -126,8 +108,7 @@ namespace BCG
 
       #endregion
 
-      private void mySubscribe(params string[] tags)
-      {
+      private void mySubscribe(params string[] tags) {
          SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
          Stream str = new MemoryStream();
          new MetaData(sub).save(str);
@@ -135,20 +116,16 @@ namespace BCG
          myConnection.send(str);
       }
 
-      private void myHandleServerReboot(string serverURL, bool connection)
-      {
-         if (connection)
-         {
+      private void myHandleServerReboot(string serverURL, bool connection) {
+         if (connection) {
             //ie connection established again, so has to
             //resubscribe to all the tags
             // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
             //    PS3LogLevels.disconnection);
             mySubscribe(myTags.ToArray());
 
-            
-         }
-         else
-         {
+
+         } else {
             // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
             //    PS3LogLevels.disconnection);
          }
@@ -158,18 +135,17 @@ namespace BCG
             onPubSubConnectionChange(connection, serverURL);
       }
 
-      internal static void processMessage(byte[] data, List<PubSubUpdate> updates)
-      {
-         if (data != null && data.Length > 0)
-         {
+      internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
+         if (data != null && data.Length > 0) {
             MemoryStream str = new MemoryStream(data);
             MetaData md = new MetaData(str);
+            bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage meta_data: {md}");
             MemoryStream bodyStream = str;
             long bodyPosition = str.Position;
-            object op = Activator.CreateInstance (Type.GetType("BCG." + md["op-code"]));
+            object op = Activator.CreateInstance(Type.GetType("BCG." + md["op-code"]));
             md.copyTo(op);
-            if (op is PubMetaData)
-            {
+            if (op is PubMetaData) {
                PubMetaData p = op as PubMetaData;
                //BCG.BlueTrace.WriteBlueCommReceiveW(p.tag, p.publishTimeTicks);
 
@@ -177,41 +153,29 @@ namespace BCG
                p.warnIfLatencyExceedsThreshold();
                //find out if we need to decompress this data
                Stream realBody = bodyStream;
-               if (p.isCompressed)
-               {
+               if (p.isCompressed) {
                   long compressedSize = bodyStream.Length - bodyStream.Position;
-                  if (compressedSize == 0)
-                  {
+                  if (compressedSize == 0) {
                      string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
                      Console.WriteLine(errorMsg);
                      //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
                      return;
-
-
                   }
-
                   realBody = StreamDecompressor.decompressStream(bodyStream);
                   realBody.Position = bodyPosition;
                }
-
-
-
                updates.Add(new PubSubUpdate(p.tag, p.type, realBody));
-            }
-            else if (op is TransactionMetaData)
-            {
+            } else if (op is TransactionMetaData) {
                TransactionMetaData tr = op as TransactionMetaData;
-               for (int i = 0; i < tr.nMessages; i++)
-               {
+               for (int i = 0; i < tr.nMessages; i++) {
                   PubMetaData pub = new PubMetaData();
                   new MetaData(bodyStream).copyTo(pub);
                   byte[] trConstituent = new byte[pub.size];
                   bodyStream.Read(trConstituent, 0, trConstituent.Length);
                   updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
                }
-            }
-            else{}
-               //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
+            } else { }
+            //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
          }
       }
    }

src/DataStreamSuite/PubSub/StaticPublisherCore.cs

@@ -15,91 +15,79 @@ using mbts;
  *       This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced. 
  *       
  * */
-namespace BCG
-{
-   public class CrushedTagManager
-   {
-      private static CrushedTagManager mySelf;
-      private Dictionary<string, bool> myTags = new Dictionary<string, bool>();
-      public static CrushedTagManager instance
-      {
-         get
-         {
-            if (mySelf == null)
-               mySelf = new CrushedTagManager();
-            return mySelf;
+namespace BCG {
+   // Provides a function to check if a tag is crushabel or not
+   public class CrushedTagManager {
+      private static CrushedTagManager _self;
+      private Dictionary<string, bool> _tag_map = new Dictionary<string, bool>();
+      public static CrushedTagManager instance {
+         get {
+            if (_self == null)
+               _self = new CrushedTagManager();
+            return _self;
          }
       }
-      public bool isTagCrushable(string tag)
-      {
+      public bool is_tag_crushable(string tag) {
          tag = tag.ToLower();
-         lock (typeof(CrushedTagManager))
-         {
-            if (!myTags.ContainsKey(tag))
-            {
+         bool rv = false;
+         lock (typeof(CrushedTagManager)) {
+            if (!_tag_map.TryGetValue(tag, out rv)) {
                foreach (string r in PubSub3Config.instance.PS3_CrushableTags)
-                  if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag))
-                  {
-                     myTags.Add(tag, true);
+                  if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) {
+                     _tag_map.Add(tag, true);
                      break;
                   }
-               if (!myTags.ContainsKey(tag))
-                  myTags.Add(tag, false);
+               _tag_map.Add(tag, false);
             }
          }
-         return myTags[tag];
+         return rv;
       }
    }
    /// <summary>
    /// All the data for a single tag, thats published within
    /// a transaction
    /// </summary>
-   public class TagWaitingInTransaction
-   {
-      private bool myIsCrushable;
-      private List<byte[]> myData = new List<byte[]>();
-      private int myTotalSize = 0;
-      private string myTag, myTagType;
-      public TagWaitingInTransaction(string theTagType, string theTag)
-      {
-         myTag = theTag;
-         myTagType = theTagType;
-         myIsCrushable = CrushedTagManager.instance.isTagCrushable(myTag);
+   public class TagWaitingInTransaction {
+      private bool _is_crushable;
+      private List<byte[]> _buffered_data = new List<byte[]>();
+      // this is the size of all the buffered data
+      private int _total_size = 0;
+      private string _tag, _tag_type;
+      public TagWaitingInTransaction(string theTagType, string theTag) {
+         _tag = theTag;
+         _tag_type = theTagType;
+         _is_crushable = CrushedTagManager.instance.is_tag_crushable(_tag);
       }
 
-      public void addPub(Stream pubStream)
-      {
-         if (myIsCrushable)
-         {
-            byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)];
-            pubStream.Read(newData, 0, newData.Length);
-            myData.Clear();
-            myData.Add(newData);
-            myTotalSize = newData.Length;
-         }
-         else
-         {
-            byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)];
-            pubStream.Read(newData, 0, newData.Length);
-            myData.Add(newData);
-            myTotalSize += newData.Length;
+      public void add_pub(Stream pub_stream) {
+         if (_is_crushable) {
+            // extract bytes, clear _buffered_data and queue to _buffered_data
+            byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
+            pub_stream.Read(new_data, 0, new_data.Length);
+            _buffered_data.Clear();
+            _buffered_data.Add(new_data);
+            _total_size = new_data.Length;
+         } else {
+            // extract btes and queue to _buffered_data
+            byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
+            pub_stream.Read(new_data, 0, new_data.Length);
+            _buffered_data.Add(new_data);
+            _total_size += new_data.Length;
          }
       }
 
-      public void writeTo(Stream s, string connection, string publishingApp)
-      {
-         new MetaData(new PubMetaData(myTagType, myTag, myTotalSize, connection, publishingApp)).save(s);
-         foreach (byte[] data in myData)
+      public void write_to(Stream s, string connection, string publishing_app) {
+         new MetaData(new PubMetaData(_tag_type, _tag, _total_size, connection, publishing_app)).save(s);
+         foreach (byte[] data in _buffered_data)
             s.Write(data, 0, data.Length);
       }
 
-      public string tag { get { return myTag; } }
-      public string tagType { get { return myTagType; } }
-      public List<byte[]> data { get { return myData; } }
+      public string tag { get { return _tag; } }
+      public string tagType { get { return _tag_type; } }
+      public List<byte[]> data { get { return _buffered_data; } }
    }
 
-   public enum PublisherCoreMessageType
-   {
+   public enum PublisherCoreMessageType {
       transactionBegin,
       transactionCommit,
       transactionAbort,
@@ -107,37 +95,35 @@ namespace BCG
       clear
    }
 
-   public class PublisherCoreMessage
-   {
+   public class PublisherCoreMessage {
       public PublisherCoreMessageType msgType;
       public string tagType, tag;
       public Stream data;
 
-      public PublisherCoreMessage(string tagToClear)
-      {
+      public PublisherCoreMessage(string tagToClear) {
          tag = tagToClear;
          msgType = PublisherCoreMessageType.clear;
 
       }
 
-      public PublisherCoreMessage(PublisherCoreMessageType theMsgType)
-      {
+      public PublisherCoreMessage(PublisherCoreMessageType theMsgType) {
          msgType = theMsgType;
       }
 
-      public PublisherCoreMessage(string theTagType, string theTag, Stream theData)
-      {
+      public PublisherCoreMessage(string theTagType, string theTag, Stream theData) {
          msgType = PublisherCoreMessageType.publish;
          tagType = theTagType;
          tag = theTag;
          data = theData;
       }
+      public override string ToString() {
+         return $"PublisherCoreMessage tag: {tag} data.Position: {data.Position} data.Length: {data.Length}";
+      }
    }
 
    // this class uses an internal queue + thread to serialize and process calls
    // made through its public interface (IPublisherCore)
-   public class StaticPublisherCore : IPublisherCore
-   {
+   public class StaticPublisherCore : IPublisherCore {
       // used to allow for different transports
       // to be used when communicating to a pubsubserver
       protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false);
@@ -172,57 +158,48 @@ namespace BCG
 
       public StaticPublisherCore()
          // TODOds
-         : this(new PubSubRouter())
-      { }
+         : this(new PubSubRouter()) { }
 
       public StaticPublisherCore(string serverURL)
-         : this(new PubSubRouter(new PubSubRoute(serverURL, ".*")))
-      { }
+         : this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) { }
 
-      public StaticPublisherCore(IPubSubRouter r)
-      {
+      public StaticPublisherCore(IPubSubRouter r) {
          myRouter = r;
          Thread pumpThread = new Thread(new ThreadStart(myMessagePump));
          pumpThread.IsBackground = true;
          pumpThread.Start();
       }
 
-      public void Dispose()
-      { }
+      public void Dispose() { }
       #endregion
 
       public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); }
 
       #region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
 
-      public void publish(string tag, Stream content)
-      { publish("", tag, content); }
+      public void publish(string tag, Stream content) { publish("", tag, content); }
 
-      public void publish(string tagType, string tag, Stream content)
-      {
-         myMainQueue.enqueue(new PublisherCoreMessage(tagType, tag, content));
+      public void publish(string tagType, string tag, Stream content) {
+         PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
+         bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}");
+         myMainQueue.enqueue(msg);
       }
 
-      public void beginTransaction()
-      {
+      public void beginTransaction() {
          myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin));
       }
 
-      public void commitTransaction()
-      {
+      public void commitTransaction() {
          myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit));
       }
 
-      public void abortTransaction()
-      {
+      public void abortTransaction() {
          myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort));
       }
 
-      public bool inTransaction
-      { get { return myInTransaction; } }
+      public bool inTransaction { get { return myInTransaction; } }
 
-      public void clearState(string tag)
-      {
+      public void clearState(string tag) {
          myMainQueue.enqueue(new PublisherCoreMessage(tag));
       }
 
@@ -230,70 +207,51 @@ namespace BCG
 
       #region Message Pump Thread - this is where messages are dequeued and the action performed
 
-      private void myMessagePump()
-      {
-         while (true)
-         {
-            try
-            {
-               while (true)
-               {
+      private void myMessagePump() {
+         while (true) {
+            try {
+               while (true) {
                   object data = myMainQueue.dequeue(1000);
-                  if (data != null)
-                  {
+                  if (data != null) {
                      int qc = myMainQueue.getCount();
                      PublisherCoreMessage msg = data as PublisherCoreMessage;
-                     if (msg.msgType == PublisherCoreMessageType.transactionBegin)
-                     {
+                     bts.Log.Gen.info($"StaticPublisherCore.myMessagePump msg: {msg}");
+                     if (msg.msgType == PublisherCoreMessageType.transactionBegin) {
                         myBeginTransaction();
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.transactionCommit)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.transactionCommit) {
                         myCommitTransaction();
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.transactionAbort)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.transactionAbort) {
                         myAbortTransaction();
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.publish)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.publish) {
                         myPublish(msg.tagType, msg.tag, msg.data);
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.clear)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.clear) {
                         myClearState(msg.tag);
                      }
                   }
 
-                  foreach (IPubSubServerConnection conn in mySenders)
-                  {
+                  foreach (IPubSubServerConnection conn in mySenders) {
                      conn.doWork();
                   }
                }
-            }
-            catch (Exception ex)
-            {
-               // BlueEventLog.logEvent(ex);
+            } catch (Exception ex) {
+               bts.Log.Gen.error($"StaticPublisherCore.myMessagePump exception ex: {ex}");
                if (ex is OutOfMemoryException)
                   throw ex;
             }
          }
       }
 
-      private void myBeginTransaction()
-      {
+      private void myBeginTransaction() {
          // BlueEventLog.logAssert(!inTransaction);
          //BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information);
          myInTransaction = true;
       }
 
-      private void myCommitTransaction()
-      {
+      private void myCommitTransaction() {
          myEndTransaction(true);
       }
 
-      private void myAbortTransaction()
-      {
+      private void myAbortTransaction() {
          myEndTransaction(false);
       }
 
@@ -304,21 +262,17 @@ namespace BCG
       //keep it consistent with the way it was done earlier), this should
       //simply publish it non transactionally (easy way out), or 
       //transactionally with respect to each pubsub
-      protected virtual void myEndTransaction(bool success)
-      {
-         lock (this)
-         {
+      protected virtual void myEndTransaction(bool success) {
+         lock (this) {
             // BlueEventLog.logAssert(inTransaction);
             //BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information);
             myInTransaction = false;
-            if (success)
-            {
+            if (success) {
                // pubsub server -> tags with updates
                Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>();
 
                // group updates per pubsub server
-               foreach (string tag in myWaitingInTransactionMessages.Keys)
-               {
+               foreach (string tag in myWaitingInTransactionMessages.Keys) {
                   string thisURL = myRouter.getPubURLForTag(tag);
                   if (!routesToTags.ContainsKey(thisURL))
                      routesToTags[thisURL] = new List<string>();
@@ -327,17 +281,14 @@ namespace BCG
 
                // more than one server means we won't be able to honor the transactionality request
                // log a warning if that is the case.
-               if (routesToTags.Keys.Count > 1)
-               {
+               if (routesToTags.Keys.Count > 1) {
                   //this is going to be a long error message, but it's an importat one to be able to debug. 
                   StringBuilder errorMessageBuilder = new StringBuilder();
                   errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n");
                   errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n");
 
-                  foreach (string route in routesToTags.Keys)
-                  {
-                     foreach (string tag in routesToTags[route])
-                     {
+                  foreach (string route in routesToTags.Keys) {
+                     foreach (string tag in routesToTags[route]) {
                         errorMessageBuilder.Append("\tTag ");
                         errorMessageBuilder.Append(tag);
                         errorMessageBuilder.Append(" is routed to ");
@@ -351,16 +302,14 @@ namespace BCG
                }
 
                // iterate through each server with pending messages
-               foreach (KeyValuePair<string, List<string>> kvp in routesToTags)
-               {
+               foreach (KeyValuePair<string, List<string>> kvp in routesToTags) {
                   IPubSubServerConnection transactionSender = null;
                   string publishingApp = ClientInfo.thisClient.processString;
                   string connection = kvp.Key;
                   Stream transactionData = new MemoryStream();
 
                   // iterate through each of the pending tag updates for one server
-                  foreach (string tag in kvp.Value)
-                  {
+                  foreach (string tag in kvp.Value) {
                      ///it would be the same transaction sender that would 
                      ///return each time
                      transactionSender = myGetOrBuildSender(tag);
@@ -372,7 +321,7 @@ namespace BCG
                         // update internal state
                         myState.publish(waiting.tag, waiting.tagType, data);
                      // add a pubsub header and copy each message to the transactionData stream
-                     waiting.writeTo(transactionData, connection, publishingApp);
+                     waiting.write_to(transactionData, connection, publishingApp);
                   }
                   // now transactionData holds all the updates for all the tags 
                   // that are to be sent to the PubSub represented by transactionSender
@@ -384,36 +333,33 @@ namespace BCG
                   transactionData.Position = 0;
 
                   // send the entire thing to the specific pubsub server.
-                  transactionSender.send(new PubSubMessage(new MetaData(tMd), transactionData).toStream());
+                  // this is where the data is actually sent out !
+                  var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData);
+                  Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}");
+                  bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
+                  transactionSender.send(ps_msg);
                }
             }
             myWaitingInTransactionMessages.Clear();
          }
       }
 
-      protected virtual void myPublish(string tagType, string tag, Stream content)
-      {
-         if (myInTransaction)
-         {
+      protected virtual void myPublish(string tagType, string tag, Stream content) {
+         bts.Log.Gen.info($"StaticPublisherCore.myPublish tag: {tag} content: {content.Position} / {content.Length} myInTransaction: {myInTransaction}");
+         if (myInTransaction) {
             ///if its in transaction message is cached, and is sent
             ///only when theres a commit or abort
             if (!myWaitingInTransactionMessages.ContainsKey(tag))
                myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag));
-            myWaitingInTransactionMessages[tag].addPub(content);
-         }
-         else
-         {
-            //if (CrushedTagManager.instance.isTagCrushable(tag))
+            myWaitingInTransactionMessages[tag].add_pub(content);
+         } else {
             {
                myState.publish(tag, tagType, content);
-               if (!myCrushableTagsWaitingOnAck.ContainsKey(tag))
-               {
-                  // BlueEventLog.logEvent("Sending crushable tag " + tag, BlueEventLevel.Diagnosis);
-                  mySendData(tag, tagType, content);
-               }
-               else
-               {
-                  // BlueEventLog.logEvent("Could not send tag " + tag + " since its waiting on an ack", BlueEventLevel.Diagnosis);
+               if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
+                  bts.Log.Gen.info("Sending crushable tag: {tag}");
+                  mySendData(tag, tagType, content, "myPublish");
+               } else {
+                  bts.Log.Gen.info("Could not send tag: {tag}  since its waiting on an ack");
                   myCrushableTagsWaitingOnAck[tag].second++;
                }
             }
@@ -428,64 +374,59 @@ namespace BCG
       }
 
       // add header, possibly compress the data, and sendto appropriate pubsub
-      private void mySendData(string tag, string tagType, Stream content)
-      {
+      private void mySendData(string tag, string tagType, Stream content, string reason) {
+         bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}");
          long initPos = content.Position;
          // locate right pubsub for this tag
          IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
          Stream contentToPublish = content;
-         PubMetaData pubMd;
+         PubMetaData pub_md;
 
-         if (PubSub3Config.instance.isTagCompressed(tag))
-         {
+         if (PubSub3Config.instance.isTagCompressed(tag)) {
             contentToPublish = new MemoryStream();
             SharpZipCompressor Kompressor = new SharpZipCompressor();
             Kompressor.compress(content, contentToPublish);
             contentToPublish.Position = 0;
-            pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
-            if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag))
-            {
+            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+            if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
                string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
                // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
             }
-         }
-         else
-            pubMd = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+         } else
+            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
 
          content.Position = initPos;
-         if (CrushedTagManager.instance.isTagCrushable(tag))
-         {
+         if (CrushedTagManager.instance.is_tag_crushable(tag)) {
             string refId = String.Empty;
             Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
-            serverConn.sendWithConfirmation((new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream() as MemoryStream).ToArray(), ref refId);
+            //  this is where the data is actually sent out (there are multiple sends)
+            PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
+            //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
+            bts.Log.Gen.info($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
+            bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
+            serverConn.send_with_confirmation(ps_msg, ref refId);
             myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
-         }
-         else
-         {
-            Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
-            serverConn.send(new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream());
+         } else {
+            // this is where the data is actually sent out
+            var ps_msg = new PubSubMessage(pub_md, contentToPublish);
+            bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
+            serverConn.send(ps_msg);
          }
       }
 
-      private void myClearState(string tag)
-      {
-         if (myTagToSender.ContainsKey(tag))
-         {
+      private void myClearState(string tag) {
+         if (myTagToSender.ContainsKey(tag)) {
             myTagToSender.Remove(tag);
          }
          myState.clear(tag);
-
       }
 
-      protected virtual IPubSubServerConnection myGetOrBuildSender(string tag)
-      {
-         if (!myTagToSender.ContainsKey(tag))
-         {
+      protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) {
+         if (!myTagToSender.ContainsKey(tag)) {
             string route = myRouter.getPubURLForTag(tag);
             IPubSubServerConnection sender = myConnectionFactory.getConnection(route);
             myTagToSender.Add(tag, sender);
-            if (!mySenders.Contains(sender))
-            {
+            if (!mySenders.Contains(sender)) {
                mySenders.Add(sender);
                sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
                sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
@@ -497,20 +438,17 @@ namespace BCG
 
       ///in case of the server coming up, it publishes all the stored state 
       ///for that server, back to it
-      protected void myHandleServerReboot(string serverURL, bool connection)
-      {
-         if (connection)
-         {
+      protected void myHandleServerReboot(string serverURL, bool connection) {
+         bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}");
+         if (connection) {
             string thisPublisherInfo = ClientInfo.thisClient.processString;
-            foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender)
-            {
-               if (p.Value.serverURL == serverURL)
-               {
+            foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
+               if (p.Value.serverURL == serverURL) {
                   string tagType;
                   string tag = p.Key;
                   myCrushableTagsWaitingOnAck.Clear();
                   byte[] data = myState.getContent(tag, out tagType);
-                  mySendData(tag, tagType, new MemoryStream(data));
+                  mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot");
 
                   //Stream content = new MemoryStream(data);
                   //Stream contentToPublish = content;
@@ -547,16 +485,12 @@ namespace BCG
       /// of waiting for ack
       /// </summary>
       private Dictionary<string, Pair<string, int>> myCrushableTagsWaitingOnAck = new Dictionary<string, Pair<string, int>>();
-      private void myOnSendConfirm(string messageId, bool confirmed)
-      {
-         foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys))
-         {
+      private void myOnSendConfirm(string messageId, bool confirmed) {
+         foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys)) {
             Pair<string, int> tagData = myCrushableTagsWaitingOnAck[tag];
-            if (messageId == tagData.first)
-            {
+            if (messageId == tagData.first) {
                myCrushableTagsWaitingOnAck.Remove(tag);
-               if (tagData.second > 0)
-               {
+               if (tagData.second > 0) {
                   // BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis);
                   /// this data was waiting on an
                   /// ack, need to resend it

src/DataStreamSuite/PubSub/StaticSubscriberCore.cs

@@ -13,8 +13,7 @@ using mbts;
 using mbts.spark;
 //using BCG.Utilities.BlueConfiguration;
 
-namespace BCG
-{
+namespace BCG {
 
    /// <summary>
    /// This class serves as the subscrition interface to the pubsub. The only methods
@@ -25,70 +24,56 @@ namespace BCG
    /// to the servers even though there might be so created by other instances of the SubscriberCore
    /// 
    /// </summary>
-   public class StaticSubscriberCore : ISubscriberCore
-   {
+   public class StaticSubscriberCore : ISubscriberCore {
       private Dictionary<string, ISingleServerSubscriberCore> myServerURLToSubCore = new Dictionary<string, ISingleServerSubscriberCore>();
       private IPubSubRouter myRouter;
       private HighPerformanceTimer myTimer = new HighPerformanceTimer();
       private PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(true);
 
       public event PubSubConnectionChange onPubSubConnectionChange;
-      
-      
+
+
       #region constructors
 
-      public StaticSubscriberCore()
-      {
+      public StaticSubscriberCore() {
          //this defaults to a basic pubsubrouter if the config key is not filled out
          myInit(myGetRouter());
       }
-      public StaticSubscriberCore(string siteName)
-      {
+      public StaticSubscriberCore(string siteName) {
          myInit(new PubSubRouter(PubSub3Config.instance.getRouting(siteName)));
       }
 
-      private IPubSubRouter myGetRouter()
-      {return new PubSubRouter(PubSub3Config.instance.getRouting());}
+      private IPubSubRouter myGetRouter() { return new PubSubRouter(PubSub3Config.instance.getRouting()); }
 
-      public StaticSubscriberCore(PubSubRoute[] routes)
-      {
+      public StaticSubscriberCore(PubSubRoute[] routes) {
          myInit(new PubSubRouter(routes));
       }
 
-      public StaticSubscriberCore(IPubSubRouter router)
-      {
+      public StaticSubscriberCore(IPubSubRouter router) {
          myInit(router);
       }
 
       #endregion
 
-      private void myInit(IPubSubRouter router)
-      {
+      private void myInit(IPubSubRouter router) {
          myRouter = router;
       }
 
-      public ArrayList getUpdates()
-      {
+      public ArrayList getUpdates() {
          return getUpdates(10);
       }
 
-      public ArrayList getUpdates(TimeSpan timeout)
-      {
+      public ArrayList getUpdates(TimeSpan timeout) {
          return getUpdates(Convert.ToInt32(timeout.TotalMilliseconds));
       }
 
-      public ArrayList getUpdates(int timeout)
-      {
+      public ArrayList getUpdates(int timeout) {
          //BCG.BlueTrace.WriteTraceMethodEnterW("StaticSubscriberCore.getUpdates");
-         lock (myServerURLToSubCore)
-         {
+         lock (myServerURLToSubCore) {
 
-            if (myServerURLToSubCore.Count == 0)
-            {
+            if (myServerURLToSubCore.Count == 0) {
                return new ArrayList();
-            }
-            else
-            {
+            } else {
                ///gets a time share for every subscriber and
                ///calls their updates
                myTimer.start();
@@ -96,6 +81,7 @@ namespace BCG
                List<PubSubUpdate> retVal = new List<PubSubUpdate>();
                foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
                   retVal.AddRange(subCore.getUpdates(timeShare));
+               _log_n_updates(retVal);
                myTimer.stop();
                //if (myTimer.lastIntervalTime > 10)
                //   BlueEventLog.logEvent("get updates taking " + myTimer.lastIntervalTime, BlueEventLevel.Error);
@@ -104,30 +90,36 @@ namespace BCG
                return new ArrayList(retVal);
             }
          }
-         
+
       }
 
-      private Dictionary<string,bool> myBlueCommTags = new Dictionary<string,bool> ();
-      
-      bool myIsBlueCommTag(string tag)
-      {
-            return false;
+      private IDictionary<string, int> _n_update_map = new Dictionary<string, int>();
+      private void _log_n_updates(IEnumerable<PubSubUpdate> updates) {
+         foreach (PubSubUpdate u in updates) {
+            if (!_n_update_map.ContainsKey(u.tag)) {
+               _n_update_map.Add(u.tag, 0);
+            }
+            _n_update_map[u.tag] = _n_update_map[u.tag] + 1;
+         }
+         // log it now
+         foreach (var kv in _n_update_map) {
+            bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}");
+         }
       }
 
-      public void subscribe(params string[] tags)
-      {
-         lock (myServerURLToSubCore)
-         {
-            foreach (string tag in tags)
-            {
-               if (myIsBlueCommTag(tag))
-               {
-               }
-               else
-               {
+      private Dictionary<string, bool> myBlueCommTags = new Dictionary<string, bool>();
+
+      bool myIsBlueCommTag(string tag) {
+         return false;
+      }
+
+      public void subscribe(params string[] tags) {
+         lock (myServerURLToSubCore) {
+            foreach (string tag in tags) {
+               if (myIsBlueCommTag(tag)) {
+               } else {
                   string serverURL = myRouter.getSubURLForTag(tag);
-                  if (!myServerURLToSubCore.ContainsKey(serverURL))
-                  {
+                  if (!myServerURLToSubCore.ContainsKey(serverURL)) {
                      // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
                      myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
                      myServerURLToSubCore[serverURL].onPubSubConnectionChange += myHandleConnectionChange;
@@ -138,28 +130,22 @@ namespace BCG
             }
          }
       }
-      public void myHandleConnectionChange (bool isConnected, string connection)
-      {
+      public void myHandleConnectionChange(bool isConnected, string connection) {
          if (onPubSubConnectionChange != null)
             onPubSubConnectionChange(isConnected, connection);
 
       }
 
-      public void unsubscribe(params string[] tags)
-      {
-         lock (myServerURLToSubCore)
-         {
+      public void unsubscribe(params string[] tags) {
+         lock (myServerURLToSubCore) {
 
-            foreach (string tag in tags)
-            {
+            foreach (string tag in tags) {
                if (myIsBlueCommTag(tag))
                   myServerURLToSubCore["bluecomm"].unsubscribe(tag);
-               else
-               {
+               else {
                   string serverURL = myRouter.getSubURLForTag(tag);
 
-                  if (!myServerURLToSubCore.ContainsKey(serverURL))
-                  {
+                  if (!myServerURLToSubCore.ContainsKey(serverURL)) {
                      // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
                      myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
                   }
@@ -168,15 +154,13 @@ namespace BCG
             }
          }
       }
-      public IPubSubRouter getRouter()
-      {
+      public IPubSubRouter getRouter() {
          return myRouter;
       }
 
       #region IDisposable Members
 
-      public void Dispose()
-      {
+      public void Dispose() {
 
          foreach (ISingleServerSubscriberCore sssc in myServerURLToSubCore.Values)
             sssc.Dispose();

src/DataStreamSuite/PubSub/StreamOpCode.cs

@@ -3,10 +3,8 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG
-{
-   public class StreamOpCode
-   {
+namespace BCG {
+   public class StreamOpCode {
       public static readonly byte create = 0,            //creates a table
                                     add = 1,             //adds a single row
                                     remove = 2,          //removes a single row

src/DataStreamSuite/PubSub/SubscriberCore.cs

@@ -4,82 +4,68 @@ using System.Linq;
 using System.Text;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
    /*
     *  This class has been changed to be a thin pass-through shell around different subscriber core implementations.
     *  The specific implementation used is determined by the config. 
     * 
     * 
     * */
-   public class SubscriberCore  : ISubscriberCore
-   {
+   public class SubscriberCore : ISubscriberCore {
       ISubscriberCore mySubscriberCore = null;
 
-      public SubscriberCore()
-      {
+      public SubscriberCore() {
          // TODOds
          mySubscriberCore = new StaticSubscriberCore();//  PubSubComponentFactory.getSubscriberCore();
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
 
-      public SubscriberCore(string siteName)
-      {
+      public SubscriberCore(string siteName) {
          mySubscriberCore = new StaticSubscriberCore(siteName);
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
-      public SubscriberCore(IPubSubRouter theRouter)
-      {
+      public SubscriberCore(IPubSubRouter theRouter) {
          // TODOds
-         mySubscriberCore = new SubscriberCore(theRouter);//  PubSubComponentFactory.getSubscriberCore();
+         mySubscriberCore = new StaticSubscriberCore(theRouter);//  PubSubComponentFactory.getSubscriberCore();
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
 
-      public SubscriberCore(PubSubRoute[] routes)
-      {
+      public SubscriberCore(PubSubRoute[] routes) {
          //this is always a static subscriber core
          mySubscriberCore = new StaticSubscriberCore(routes);
          mySubscriberCore.onPubSubConnectionChange += myHandleConnectionChange;
       }
 
 
-      public IPubSubRouter getRouter()
-      {
+      public IPubSubRouter getRouter() {
          return mySubscriberCore.getRouter();
       }
 
-      public ArrayList getUpdates()
-      {
+      public ArrayList getUpdates() {
          return mySubscriberCore.getUpdates(10);
       }
 
-      public ArrayList getUpdates(TimeSpan timeout)
-      {
+      public ArrayList getUpdates(TimeSpan timeout) {
          return mySubscriberCore.getUpdates(Convert.ToInt32(timeout.TotalMilliseconds));
       }
 
-      public ArrayList getUpdates(int timeout)
-      {
+      public ArrayList getUpdates(int timeout) {
          return mySubscriberCore.getUpdates(timeout);
       }
 
-      public void subscribe(params string[] tags)
-      {
+      public void subscribe(params string[] tags) {
          mySubscriberCore.subscribe(tags);
       }
 
-      public void unsubscribe(params string[] tags)
-      {
+      public void unsubscribe(params string[] tags) {
          mySubscriberCore.unsubscribe(tags);
       }
 
-      public void Dispose()
-      {
+      public void Dispose() {
          mySubscriberCore.Dispose();
 
       }
-      private void myHandleConnectionChange(bool isConnected, string connection)
-      {
+      private void myHandleConnectionChange(bool isConnected, string connection) {
          if (onPubSubConnectionChange != null)
             onPubSubConnectionChange(isConnected, connection);
       }

src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs

@@ -0,0 +1,132 @@
+using System;
+using System.Linq.Expressions;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading.Tasks;
+using System.IO;
+
+using Spark.Core.Reactive;
+using System.Security.Cryptography.X509Certificates;
+using BCG;
+
+namespace BCG
+{
+
+   // This uses AsyncTCPConnection2's stateful async functions
+   public class SimpleTCPClientAsync : ITCPConnectionHandler, ITCPMessageHandler
+   {
+      AsyncTCPConnection2 _conn = null;
+      int _m_msgs_received = 0;
+      public static async Task start(string ip, int port)
+      {
+         SimpleTCPClientAsync c = new SimpleTCPClientAsync(ip, port);
+         c.connect();
+      }
+
+      public void connect()
+      {
+         Task.Run(async () => await TCPSocketUtils.connect_to_server_async("127.0.0.1", 12345, this, TimeSpan.FromSeconds(2)));
+      }
+
+      public void onConnection(ITCPConnection connection)
+      {
+         Console.WriteLine("SimpleTCPClientAsync.onConnection connected to server, starting task to send data in a loop");
+         connection.register(this);
+         // start sending data
+         Task.Run(async () =>
+         {
+            byte[] buff = new byte[1024];
+            for (int i = 0; i < buff.Length; i++)
+            {
+               buff[i] = (byte)i;
+            }
+            _conn = connection as AsyncTCPConnection2;
+            while (!_conn.isDisposed)
+            {
+               await _conn.send_async(buff);
+               await Task.Delay(100);
+            }
+         });
+      }
+
+      public void onData(RemoteMessage msg)
+      {
+         _m_msgs_received++;
+         Console.WriteLine($"SimpleTCPClientAsync.onData n_msgs: {_m_msgs_received} msg: {msg}");
+         if (msg.messageType == RemoteMessageType.Disconnection)
+         {
+            connect();
+         }
+      }
+
+      public bool isDisposed { get { return false; } }
+
+      public void Dispose() { }
+
+      public SimpleTCPClientAsync(string ip, int port) { }
+
+   }
+   public class SimpleTCPClient : ITCPConnectionHandler, ITCPMessageHandler
+   {
+      private int _n_msgs_received = 0;
+      AsyncTCPConnection2 _conn = null;
+
+      public static void start(string ip, int port)
+      {
+         SimpleTCPClient c = new SimpleTCPClient(ip, port);
+         c.connect();
+      }
+
+      public void connect()
+      {
+         Console.WriteLine("SimpleTCPClient.onConnection connect() called");
+         ClientToServerManagedTCPConnector.connect_to(new BlutoSubAddress("bluto://127.0.0.1:12345"), this, TimeSpan.FromSeconds(1));
+      }
+
+      public void onConnection(ITCPConnection connection)
+      {
+         _conn = connection as AsyncTCPConnection2;
+         Console.WriteLine("SimpleTCPClient.onConnection connected to server, starting task to send data in a loop");
+         _conn.register(this);
+         byte[] buff = new byte[1024];
+         for (int i = 0; i < buff.Length; i++)
+         {
+            buff[i] = (byte)i;
+         }
+
+
+         while (!_conn.isDisposed)
+         {
+            _conn.send(buff);
+            System.Threading.Thread.Sleep(1000);
+         }
+
+         // Task.Run(async () =>
+         // {
+         //    while (!_conn.isDisposed)
+         //    {
+         //       await _conn.send_async(buff);
+         //       await Task.Delay(100);
+         //    }
+         // });
+      }
+
+      public void onData(RemoteMessage msg)
+      {
+         _n_msgs_received++;
+         Console.WriteLine($"SimpleTCPClient.onData messages received: {_n_msgs_received} msg: {msg} messages received: {_n_msgs_received}");
+         if (msg.messageType == RemoteMessageType.Disconnection)
+         {
+            connect();
+         }
+      }
+
+      public bool isDisposed { get { return false; } }
+
+      public void Dispose() { }
+
+      public SimpleTCPClient(string ip, int port) { }
+
+   }
+}

src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs

@@ -0,0 +1,123 @@
+using BCG;
+
+using Google.Protobuf;
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+using System.Xml.Serialization;
+using System.Reflection;
+using System.Data;
+using System.Collections.Generic;
+
+namespace BCG
+{
+
+   public class SimpleTCPServerAsync : ITCPConnectionHandler, ITCPMessageHandler
+   {
+      int _n_msgs_received = 0;
+      SingleServerToClientManagedTCPConnectionAcceptor _acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+      IDictionary<RemoteConnectionInfo, ITCPConnection> _conn_map = new Dictionary<RemoteConnectionInfo, ITCPConnection>();
+
+      public static async Task start()
+      {
+         Console.WriteLine("SimpleTCPServer.start accepting socket connections");
+
+         SimpleTCPServerAsync s = new SimpleTCPServerAsync();
+         //await AsyncTCPConnection2.accept_connections("127.0.0.1", 12345, s, TimeSpan.FromMilliseconds(100));
+         s.accept();
+      }
+
+      // public static async Task init()
+      // {
+      // }
+
+      public void accept()
+      {
+         _acceptor.accept_on_port(this, "127.0.0.1", 12345, TimeSpan.Zero);
+      }
+
+      public void onConnection(ITCPConnection connection)
+      {
+         AsyncTCPConnection2 new_conn = connection as AsyncTCPConnection2;
+         RemoteConnectionInfo rci = new_conn.get_peer_address();
+         _conn_map.Add(rci, new_conn);
+         Console.WriteLine($"SimpleTCPServerAsync.onConnection new client connected conn: {rci}");
+         new_conn.register(this);
+      }
+
+
+      public void onData(RemoteMessage msg)
+      {
+         Console.WriteLine($"SimpleTCPServerAsync.onData msg: {msg}");
+         RemoteConnectionInfo rci = msg.remoteConnectionInfo;
+         ITCPConnection this_conn;
+         bool found = _conn_map.TryGetValue(rci, out this_conn);
+         if (!found)
+         {
+            Console.WriteLine("Could not find connection !!!!!!");
+            return;
+         }
+         _n_msgs_received++;
+         Console.WriteLine($"SimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} echoing it back");
+         if (msg.messageType == RemoteMessageType.Data)
+         {
+
+            this_conn.send(msg.message);
+            // Task.Run(async () => {
+            //    Console.WriteLine($"SimpleTCPServerAsync.onData sending msg: {msg} to client");
+            //    await _conn.send_async(msg.message);
+            // });
+         }
+      }
+
+      public bool isDisposed { get { return false; } }
+
+      public void Dispose() { }
+   }
+
+   //    public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler {
+   //       private AsyncTCPConnection2 _conn = null;
+   //       private int _n_msgs_received = 0;
+   //       SingleServerToClientManagedTCPConnectionAcceptor _acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+
+   //       public static async Task start() {
+   //          Console.WriteLine("SimpleTCPServer.start accepting socket connections");
+   //          SimpleTCPServer s = new SimpleTCPServer();
+   //          s.accept();
+   //          //await AsyncTCPConnection2.accept_connections("127.0.0.1", 12345, s, TimeSpan.FromMilliseconds(100));
+   //       }
+
+   //       public void accept() {
+   //          _acceptor.accept_on_port(this, "127.0.0.1", 12345, TimeSpan.Zero);
+   //       }
+
+   //       public void onConnection(ITCPConnection connection) {
+   //          Console.WriteLine("SimpleTCPServerAsync.onConnection new client connected");
+   //          _conn = connection as AsyncTCPConnection2;
+   //          connection.register(this);
+   //       }
+
+   //       public async void onData(RemoteMessage msg) {
+   //          _n_msgs_received++;
+   //          Console.WriteLine($"SimpleTCPServerAsync.onData msg: {msg} , msgs received: {_n_msgs_received} echoing it back");
+   //          if (msg.messageType == RemoteMessageType.data && _conn != null) {
+   //             Task.Run(async () => {
+   //                //Console.WriteLine($"SimpleTCPServerAsync.onData sending msg: {msg} to client");
+   //                //await _conn.send_async(msg.message);
+   //                _conn.send(msg.message);
+   //             });
+   //          }
+   //       }
+
+   //       public bool isDisposed { get { return false; } }
+
+   //       public void Dispose() { }
+   //    }
+}