Fixing mangled merge

src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs

@@ -1,72 +1,68 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
 
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
-   /// This class will be used by the clients to publish data to the pubsub
+   /// This class will be used by the clients to publish tabular data to the pubsub
    /// </summary>
-   public class PubTableBroker 
-   {
-      protected PublisherCore myPublisherCore;
-      protected IDictionary myPublisherMap = new Hashtable ();
-      protected IDictionary mySchemaMap = new Hashtable ();
-      
+   public class PubTableBroker {
+      protected PublisherCore _publisher_core;
+      protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
+      protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
+
       protected bool myWaitingForFirstOp = false;
 
-      public PubTableBroker ()  
-         : this (new PublisherCore()) 
-      {}
+      public PubTableBroker()
+         : this(new PublisherCore()) { }
 
-      public PubTableBroker (PublisherCore publisher)   
-      {myPublisherCore = publisher;}
+      public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
 
-      public PubTableBroker (IBlueRowCollection[] tablesToPublish) 
-         : this (new PublisherCore(), tablesToPublish)
-      {}
+      public PubTableBroker(IBlueRowCollection[] tablesToPublish)
+         : this(new PublisherCore(), tablesToPublish) { }
 
-      public PubTableBroker (PublisherCore publisher, IBlueRowCollection[] tablesToPublish)
-      {
-         myPublisherCore = publisher;
+      public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
+         _publisher_core = publisher;
 
-         myPublisherCore.beginTransaction ();
+         _publisher_core.beginTransaction();
          foreach (IBlueRowCollection table in tablesToPublish)
-            createTable (table);
-         myPublisherCore.commitTransaction ();
+            createTable(table);
+         _publisher_core.commitTransaction();
       }
 
-      public void beginTransaction ()
-      {
+      public void beginTransaction() {
          myWaitingForFirstOp = true;
       }
 
-      public void commitTransaction ()
-      {
+      public void commitTransaction() {
+         bts.Log.Gen.info("PubTableBroker.commitTransaction");
          if (myWaitingForFirstOp)
             myWaitingForFirstOp = false;
          else
-            myPublisherCore.commitTransaction ();
+            _publisher_core.commitTransaction();
       }
 
-      public void rollbackTransaction ()
-      {
+      public void rollbackTransaction() {
+         bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
          if (myWaitingForFirstOp)
             myWaitingForFirstOp = false;
          else
-            myPublisherCore.abortTransaction ();
+            _publisher_core.abortTransaction();
       }
 
-      public virtual void createTable (IBlueRowCollection newTable)
-      {
+      public virtual void createTable(IBlueRowCollection newTable) {
+         bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
          myCheckTransaction();
-         if (myPublisherMap [newTable.schema.tableName.ToLower()] == null)
-         {
-            myPublisherMap [newTable.schema.tableName.ToLower()] = myGetIPubTablePublisher ();
-            mySchemaMap [newTable.schema.tableName.ToLower()] = newTable.schema;
+         IPubTablePublisher table_publisher = null;
+         var name_key = newTable.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
+            table_publisher = _create_pubtable_publisher(name_key);
+            _publisher_map.Add(name_key, table_publisher);
+            _schema_map.Add(name_key, newTable.schema);
          }
-         (myPublisherMap[newTable.schema.tableName.ToLower()] as IPubTablePublisher).createTable (newTable);
+         table_publisher.createTable(newTable);
       }
 
       /// <summary>
@@ -74,13 +70,14 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void addRow (IBlueRow r)
-      {
+      public virtual void addRow(IBlueRow r) {
          myCheckTransaction();
-         if (myPublisherMap[r.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Row can not be added, as the table that has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = r.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Row can not be added, as the table that has not been created yet");
          else
-            (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).addRow (r);
+            table_publisher.addRow(r);
       }
 
       /// <summary>
@@ -88,13 +85,14 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void deleteRow (IBlueRow r)
-      {
+      public virtual void deleteRow(IBlueRow r) {
          myCheckTransaction();
-         if (myPublisherMap[r.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Row can not be removed, as the table has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = r.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Row can not be removed, as the table has not been created yet");
          else
-            (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).deleteRow (r);
+            table_publisher.deleteRow(r);
       }
 
       /// <summary>
@@ -102,13 +100,14 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void changeRow (IBlueRow r)
-      {
+      public virtual void changeRow(IBlueRow r) {
          myCheckTransaction();
-         if (myPublisherMap[r.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Row can not be changed, as the table has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = r.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Row can not be changed, as the table has not been created yet");
          else
-            (myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).changeRow (r);
+            table_publisher.changeRow(r);
       }
 
       /// <summary>
@@ -116,40 +115,37 @@ namespace BCG
       /// will be thrown
       /// </summary>
       /// <param name="r"></param>
-      public virtual void makeBatchChanges (IBlueRowCollection newRows, 
-                                             IBlueRowCollection deletedRows, 
-                                             IBlueRowCollection addedRows)
-      {
+      public virtual void makeBatchChanges(IBlueRowCollection newRows,
+                                             IBlueRowCollection deletedRows,
+                                             IBlueRowCollection addedRows) {
          myCheckTransaction();
-         if (myPublisherMap[newRows.schema.tableName.ToLower()] == null)
-            throw new ApplicationException ("Batch changes can not be made, as the table has not been created yet");
+         IPubTablePublisher table_publisher = null;
+         var name_key = newRows.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
          else
-            (myPublisherMap [newRows.schema.tableName.ToLower()] as IPubTablePublisher).batchRemoveAdd (deletedRows, addedRows);
+            table_publisher.batchRemoveAdd(deletedRows, addedRows);
       }
 
-      public virtual void deleteTableRows (string tableName)
-      {
+      public virtual void deleteTableRows(string tableName) {
          myCheckTransaction();
-         (myPublisherMap [tableName.ToLower()] as IPubTablePublisher).createTable (
-            new BlueTable (mySchemaMap[tableName.ToLower()] as IBlueTableSchema));
+         (_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
+            new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
       }
 
-      public int getNumberOfWaitingMessages() { return myPublisherCore.getNumberOfWaitingMessages(); }
-   
-      public void clearTable(string tableName)
-      {myPublisherCore.clearState(PubStreamPrefixer.prefix(tableName));}
+      public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
 
-      protected void myCheckTransaction()
-      {
+      public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
+
+      protected void myCheckTransaction() {
          if (myWaitingForFirstOp)
-            myPublisherCore.beginTransaction ();
+            _publisher_core.beginTransaction();
          myWaitingForFirstOp = false;
       }
 
-      protected virtual IPubTablePublisher myGetIPubTablePublisher ()
-      {
-         return new PubTablePublisher (myPublisherCore);
+      protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
+         return new PubTablePublisher(tag, _publisher_core);
       }
 
    }
-   }
+}

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -4,62 +4,51 @@ using System.Data;
 
 using BCG.Utilities;
 
-namespace BCG
-{
-	public class ViewPublisher : IDisposable
-	{
-      protected ViewPublisher () {}
+namespace BCG {
+   public class ViewPublisher : IDisposable {
+      protected ViewPublisher() { }
 
       /// If an name IsNullOrEmpty then it uses the name
       /// in the schema (be careful with that because this
       /// could lead to double publishes if the schema name
       /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker br, ViewStream vs)
-      {
+      public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
          myName = name;
-         myView = new TableStream(name); // TODOds String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
+         myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
          myBroker = br;
          myFirstTime = true;
          myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      public void Dispose()
-      {
+      public void Dispose() {
          myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
          if (!String.IsNullOrEmpty(myName))
             myBroker.clearTable(myName);
       }
 
-      private void myChangeHandler(object o)
-      {
-         DataPacket dp = (DataPacket) o;
-         if (myFirstTime) 
-         {
+      private void myChangeHandler(object o) {
+         DataPacket dp = (DataPacket)o;
+         if (myFirstTime) {
             myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
             myLastRowCount = dp.newRows.Count;
             myBroker.createTable(dp.newRows);
             myFirstTime = false;
-         }
-         else 
-         {
-            if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) 
-            {
-               myLastRowCount  = dp.newRows.Count;
-               myBroker.makeBatchChanges(dp.newRows,dp.deletedRows, dp.addedRows);
+         } else {
+            if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
+               myLastRowCount = dp.newRows.Count;
+               myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
             }
          }
       }
 
-      public int getLastRowPublishedCount()
-      {return myLastRowCount;}
+      public int getLastRowPublishedCount() { return myLastRowCount; }
 
-      public bool hasPublished()
-      { return !myFirstTime; }
+      public bool hasPublished() { return !myFirstTime; }
 
       private int myLastRowCount = -1;
       private string myName;
       private PubTableBroker myBroker;
       protected ViewStream myView;
       private bool myFirstTime;
-	}
+   }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -112,6 +112,34 @@ namespace BCG
 
       #endregion
 
+      public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width)
+      {
+         // Print column headers
+         Console.WriteLine($"{table.schema.tableName}______________________________________");
+         foreach (var col_name in table.schema.columnNames)
+         {
+            int col_width = get_col_width(col_name);
+            Console.Write($"{col_name.PadRight(col_width)}  ");
+         }
+         Console.WriteLine();
+         Console.WriteLine("______________________________________");
+
+         // Print table data
+         foreach (IBlueRow row in table)
+         {
+            foreach (string col_name in table.schema.columnNames)
+            {
+               int col_width = get_col_width(col_name);
+               object row_val = row[col_name];
+               string row_val_str = row_val.ToString();
+               if (row_val != null)
+               {
+                  Console.Write($"{row_val_str.PadRight(col_width)} ");
+               }
+            }
+            Console.WriteLine();
+         }
+      }
       private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
       {
          { TypeCode.Boolean, typeof(bool) },
@@ -895,12 +923,12 @@ namespace BCG
                                                   out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows)
       {
          Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) =>
-            {
-               foreach (int col_no in col_indices)
-                  if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
-                     return false;
-               return true;
-            };
+         {
+            foreach (int col_no in col_indices)
+               if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
+                  return false;
+            return true;
+         };
 
          BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema);
          // adds + changes

src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs

@@ -134,18 +134,17 @@ namespace BCG
       /// <param name="row"></param>
       public virtual void addRow(IBlueRow row)
       {
-         object val = row["id"];
+         object id = row["id"];
 
-         if (val == null || val == DBNull.Value)
+         if (id == null || id == DBNull.Value)
             throw new BlueDataException("The ID of the row being added is null/DNBull");
          try
          {
-            myIdIndex.Add(val, row);
+            myIdIndex.Add(id, row);
          }
          catch (ArgumentException)
          {
-            throw new BlueDataException("ID already exists in the table " + mySchema.tableName +
-               " , duplication of field ID not allowed");
+            throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed");
          }
 
       }

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -1,316 +1,316 @@
-using System;
-using System.Collections;
-using System.Data;
-using BCG.Utilities.General;
+// using System;
+// using System.Collections;
+// using System.Data;
+// using BCG.Utilities.General;
 
-namespace BCG
-{
-	[Serializable]
-   public class DataPacket : ICloneable, IDataPacket
-   {      
-      public class BlueRowIdComparator
-      {
-         private string[] myIdColumns = new string[]{"id"};
+// namespace BCG
+// {
+// 	[Serializable]
+//    public class DataPacket : ICloneable, IDataPacket
+//    {      
+//       public class BlueRowIdComparator
+//       {
+//          private string[] myIdColumns = new string[]{"id"};
 
-         public BlueRowIdComparator(string[] idCols)
-         {
-            if ( idCols.Length != 0 ) 
-            {
-               myIdColumns = idCols;
-            }
-         }
-                    
-         public  bool blueRowIdEqualEvaluator(object a, object b)
-         {
-            if ( a == null || b == null )
-            {
-               return false;
-            }
+//          public BlueRowIdComparator(string[] idCols)
+//          {
+//             if ( idCols.Length != 0 ) 
+//             {
+//                myIdColumns = idCols;
+//             }
+//          }
 
-            if ( !(a is IBlueRow) || !(b is IBlueRow) )
-            {
-               return false;
-            }
+//          public  bool blueRowIdEqualEvaluator(object a, object b)
+//          {
+//             if ( a == null || b == null )
+//             {
+//                return false;
+//             }
 
-            IBlueRow lhs = (IBlueRow)a;
-            IBlueRow rhs = (IBlueRow)b;
-                    
-            bool result = true;
-            foreach(string idColumn in myIdColumns)
-            {
-               result = result && lhs[idColumn].Equals(rhs[idColumn]);
-               if ( !result )
-                  break;
-            }
+//             if ( !(a is IBlueRow) || !(b is IBlueRow) )
+//             {
+//                return false;
+//             }
 
-            return result;            
-         }
-      }
+//             IBlueRow lhs = (IBlueRow)a;
+//             IBlueRow rhs = (IBlueRow)b;
 
-      public IBlueRowCollection newRows, addedRows, deletedRows;
-      public DataPacket () {}
-      public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
-      {
-         this.newRows = newRows;
-         this.addedRows = addedRows;
-         this.deletedRows = deletedRows;
-      }
+//             bool result = true;
+//             foreach(string idColumn in myIdColumns)
+//             {
+//                result = result && lhs[idColumn].Equals(rhs[idColumn]);
+//                if ( !result )
+//                   break;
+//             }
 
-      public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
-      {
-         new_rows = newRows;
-         added_rows = addedRows;
-         deleted_rows = deletedRows;
-      }
+//             return result;            
+//          }
+//       }
 
-      public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
-      {                 
-         BlueTable truly = new BlueTable(addedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+//       public IBlueRowCollection newRows, addedRows, deletedRows;
+//       public DataPacket () {}
+//       public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
+//       {
+//          this.newRows = newRows;
+//          this.addedRows = addedRows;
+//          this.deletedRows = deletedRows;
+//       }
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
+//       {
+//          new_rows = newRows;
+//          added_rows = addedRows;
+//          deleted_rows = deletedRows;
+//       }
 
-         ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
-           
-         foreach(IBlueRow r in ta)
-         {
-            truly.addRow(r);
-         }
-           
-         return truly;
-                
-      }
+//       public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
+//       {                 
+//          BlueTable truly = new BlueTable(addedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-      public virtual IBlueRowCollection getTrulyAddedRows ()
-      {
-         BlueTable retVal = new BlueTable (schema);
-         foreach (IBlueRow r in addedRows)
-            if (!deletedRows.containsID ((int)r["id"]))
-               retVal.addRow (r);
-         return retVal;
-      }
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-      public virtual IBlueRowCollection getTrulyDeletedRows ()
-      {
-         BlueTable retVal = new BlueTable (schema);
-         foreach (IBlueRow r in deletedRows)
-            if (!addedRows.containsID ((int)r["id"]))
-               retVal.addRow (r);
-         return retVal;
-      }
+//          ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
 
-      public virtual IBlueRowCollection getTrulyChangedRows ()
-      {
-         BlueTable retVal = new BlueTable (schema);
-         foreach (IBlueRow r in addedRows)
-            if (deletedRows.containsID ((int)r["id"]))
-               retVal.addRow (r);
-         return retVal;
-      }
+//          foreach(IBlueRow r in ta)
+//          {
+//             truly.addRow(r);
+//          }
 
-      
-      public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
-      {        
-         BlueTable truly = new BlueTable(deletedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+//          return truly;
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       }
 
-         ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
-                      
-         foreach(IBlueRow r in td)
-         {
-            truly.addRow(r);
-         }
+//       public virtual IBlueRowCollection getTrulyAddedRows ()
+//       {
+//          BlueTable retVal = new BlueTable (schema);
+//          foreach (IBlueRow r in addedRows)
+//             if (!deletedRows.containsID ((int)r["id"]))
+//                retVal.addRow (r);
+//          return retVal;
+//       }
 
-         return truly;      
-      }
-      
-      public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
-      {
-       
-         BlueTable truly = new BlueTable(addedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
+//       public virtual IBlueRowCollection getTrulyDeletedRows ()
+//       {
+//          BlueTable retVal = new BlueTable (schema);
+//          foreach (IBlueRow r in deletedRows)
+//             if (!addedRows.containsID ((int)r["id"]))
+//                retVal.addRow (r);
+//          return retVal;
+//       }
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public virtual IBlueRowCollection getTrulyChangedRows ()
+//       {
+//          BlueTable retVal = new BlueTable (schema);
+//          foreach (IBlueRow r in addedRows)
+//             if (deletedRows.containsID ((int)r["id"]))
+//                retVal.addRow (r);
+//          return retVal;
+//       }
 
-         ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
-         ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
-         
-         foreach(IBlueRow r in tc)
-         {
-            truly.addRow(r);
-         }   
-        
-         return truly;      
-      }
-      
-      public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
-      {        
-         BlueTable truly = new BlueTable(deletedRows.schema);
-         BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
+//       {        
+//          BlueTable truly = new BlueTable(deletedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-         ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
-         ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
-         
-         foreach(IBlueRow r in tco)
-         {
-            truly.addRow(r);
-         }         
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-         return truly;         
-      }
-      
-      public TablePacket toTablePacket ()
-      {
-         return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
-                                 BlueDataUtility.toDataTable (deletedRows));
-                                 
-      }
+//          ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
 
-      public IBlueTableSchema schema 
-      {
-         get
-         {
-            IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
-               addedRows.schema != null ? addedRows.schema : deletedRows.schema;
-            return retVal;
-         }
-      }
+//          foreach(IBlueRow r in td)
+//          {
+//             truly.addRow(r);
+//          }
 
-      /*          
-       status  D total  A total  D this   A This   D' total A' Total 
-         x  0  0  0  0  0  0  
-            0  0  0  1     1 this+  
-            0  0  1  0  1this    
-            0  0  1  1  1 this   1 this+  
-         x  0  1  0  0        
-         error 0  1  0  1        
-            0  1  1  0     1 total- 
-            0  1  1  1     1 total-, 1 this +   
-         x  1  0  0  0        
-            1  0  0  1     1 this+  
-         error 1  0  1  0        
-         error 1  0  1  1        
-         x  1  1  0  0        
-         error 1  1  0  1        
-            1  1  1  0     1 total -1  keep the old delete and remove the old add
-            1  1  1  1     1 total-, 1 this +   
-      
-         d' = dthis - (dthis intersect a) + d // keep the old delete
-         a' = a - (dthis intersect a) + athis // keep the new add       
-       * */
-      public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
-      {       
-         if ( packets.Count == 0 )
-         {
-            return new DataPacket(
-               new BlueTable(new DataTable()),
-               new BlueTable(new DataTable()),
-               new BlueTable(new DataTable()));                    
-         }
+//          return truly;      
+//       }
 
-         DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
-         
-         CollectionHelpers.EqualDelegate ed = 
-            new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+//       public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
+//       {
 
-         ArrayList added = new ArrayList();
-         ArrayList deleted = new ArrayList();
-         BlueTable dt = null;
-         BlueTable at = null;
-         IBlueRowCollection nt = null;
+//          BlueTable truly = new BlueTable(addedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-         foreach(DataPacket dp in packets)
-         {               
-            ArrayList oldDeleted = deleted;
-            ArrayList oldAdded = added;
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-            added = new ArrayList();
-            deleted = new ArrayList();
-            if ( dt == null )
-            {
-               dt = new BlueTable(dp.schema);
-               at = new BlueTable(dp.schema);
-            }
-            
-            deleted.AddRange(oldDeleted);
-            deleted.AddRange( CollectionHelpers.difference(
-               CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
-               oldDeleted, ed) );
+//          ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
+//          ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
 
-            added.AddRange(dp.addedRows);
-            added.AddRange( CollectionHelpers.difference(
-               CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
-               added, ed) );
-            nt = dp.newRows;
-         }        
-         
-         foreach ( IBlueRow dr in added )
-         {
-            at.addRow(dr);
-         }
+//          foreach(IBlueRow r in tc)
+//          {
+//             truly.addRow(r);
+//          }   
 
-         foreach ( IBlueRow dr in deleted )
-         {
-            dt.addRow(dr);
-         }
+//          return truly;      
+//       }
 
-         return new DataPacket(nt,at,dt);            
-      }
+//       public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
+//       {        
+//          BlueTable truly = new BlueTable(deletedRows.schema);
+//          BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
 
-      public DataPacket deepCopy ()
-      {
-         return new DataPacket (new OrderedBlueTable (newRows),
-            new OrderedBlueTable (newRows),
-            new BlueTable (deletedRows));
-      }
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
 
-      public virtual DataPacket createImmutable ()
-      {
-         return 
-         new DataPacket(
-               newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
-               addedRows, 
-               deletedRows);
-      }
-      #region ICloneable Members
+//          ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
+//          ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
 
-      public object Clone()
-      {
-         return MemberwiseClone();
-      }
+//          foreach(IBlueRow r in tco)
+//          {
+//             truly.addRow(r);
+//          }         
 
-      #endregion
+//          return truly;         
+//       }
 
-      public string toIncrementalIDString()
-      {
-         System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
-         sb.Append ("\ndeleted : ");
-         foreach (IBlueRow r in deletedRows)
-         {
-            sb.Append(r["id"]);
-            sb.Append(',');
-         }
-         sb.Append("\nadded : ");
-         foreach (IBlueRow r in addedRows)
-         {
-            sb.Append(r["id"]);
-            sb.Append(',');
-         }
-         sb.Append("\n");
-         return sb.ToString();
-      }
+//       public TablePacket toTablePacket ()
+//       {
+//          return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
+//                                  BlueDataUtility.toDataTable (deletedRows));
 
-      public override string ToString()
-      {
-         return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
-      }
-   }
-}
+//       }
+
+//       public IBlueTableSchema schema 
+//       {
+//          get
+//          {
+//             IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
+//                addedRows.schema != null ? addedRows.schema : deletedRows.schema;
+//             return retVal;
+//          }
+//       }
+
+//       /*          
+//        status  D total  A total  D this   A This   D' total A' Total 
+//          x  0  0  0  0  0  0  
+//             0  0  0  1     1 this+  
+//             0  0  1  0  1this    
+//             0  0  1  1  1 this   1 this+  
+//          x  0  1  0  0        
+//          error 0  1  0  1        
+//             0  1  1  0     1 total- 
+//             0  1  1  1     1 total-, 1 this +   
+//          x  1  0  0  0        
+//             1  0  0  1     1 this+  
+//          error 1  0  1  0        
+//          error 1  0  1  1        
+//          x  1  1  0  0        
+//          error 1  1  0  1        
+//             1  1  1  0     1 total -1  keep the old delete and remove the old add
+//             1  1  1  1     1 total-, 1 this +   
+
+//          d' = dthis - (dthis intersect a) + d // keep the old delete
+//          a' = a - (dthis intersect a) + athis // keep the new add       
+//        * */
+//       public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
+//       {       
+//          if ( packets.Count == 0 )
+//          {
+//             return new DataPacket(
+//                new BlueTable(new DataTable()),
+//                new BlueTable(new DataTable()),
+//                new BlueTable(new DataTable()));                    
+//          }
+
+//          DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
+
+//          CollectionHelpers.EqualDelegate ed = 
+//             new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
+
+//          ArrayList added = new ArrayList();
+//          ArrayList deleted = new ArrayList();
+//          BlueTable dt = null;
+//          BlueTable at = null;
+//          IBlueRowCollection nt = null;
+
+//          foreach(DataPacket dp in packets)
+//          {               
+//             ArrayList oldDeleted = deleted;
+//             ArrayList oldAdded = added;
+
+//             added = new ArrayList();
+//             deleted = new ArrayList();
+//             if ( dt == null )
+//             {
+//                dt = new BlueTable(dp.schema);
+//                at = new BlueTable(dp.schema);
+//             }
+
+//             deleted.AddRange(oldDeleted);
+//             deleted.AddRange( CollectionHelpers.difference(
+//                CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
+//                oldDeleted, ed) );
+
+//             added.AddRange(dp.addedRows);
+//             added.AddRange( CollectionHelpers.difference(
+//                CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
+//                added, ed) );
+//             nt = dp.newRows;
+//          }        
+
+//          foreach ( IBlueRow dr in added )
+//          {
+//             at.addRow(dr);
+//          }
+
+//          foreach ( IBlueRow dr in deleted )
+//          {
+//             dt.addRow(dr);
+//          }
+
+//          return new DataPacket(nt,at,dt);            
+//       }
+
+//       public DataPacket deepCopy ()
+//       {
+//          return new DataPacket (new OrderedBlueTable (newRows),
+//             new OrderedBlueTable (newRows),
+//             new BlueTable (deletedRows));
+//       }
+
+//       public virtual DataPacket createImmutable ()
+//       {
+//          return 
+//          new DataPacket(
+//                newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
+//                addedRows, 
+//                deletedRows);
+//       }
+//       #region ICloneable Members
+
+//       public object Clone()
+//       {
+//          return MemberwiseClone();
+//       }
+
+//       #endregion
+
+//       public string toIncrementalIDString()
+//       {
+//          System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
+//          sb.Append ("\ndeleted : ");
+//          foreach (IBlueRow r in deletedRows)
+//          {
+//             sb.Append(r["id"]);
+//             sb.Append(',');
+//          }
+//          sb.Append("\nadded : ");
+//          foreach (IBlueRow r in addedRows)
+//          {
+//             sb.Append(r["id"]);
+//             sb.Append(',');
+//          }
+//          sb.Append("\n");
+//          return sb.ToString();
+//       }
+
+//       public override string ToString()
+//       {
+//          return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs

@@ -2,171 +2,151 @@ using System;
 using System.Collections;
 using System.IO;
 
-namespace BCG
-{
-	/// <summary>
-	/// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer 
-	/// that it initializes according to the table, to optimize serialiazation, hence bw and performance
-	/// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
-	/// ie the first call will always be create.
-	/// </summary>
-	public class PubTablePublisher : IPubTablePublisher
-	{
+namespace BCG {
+   /// <summary>
+   /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer 
+   /// that it initializes according to the table, to optimize serialiazation, hence bw and performance
+   /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
+   /// ie the first call will always be create.
+   /// </summary>
+   public class PubTablePublisher : IPubTablePublisher {
+      private readonly string _tag;
       protected IBlueDataSerializer mySerializer;
       protected PublisherCore myPubCore;
-		
-      public PubTablePublisher (PublisherCore publisherCore) 
-      {
+
+      public PubTablePublisher(string tag, PublisherCore publisherCore) {
+         _tag = tag;
+         bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}");
          myPubCore = publisherCore;
-         mySerializer = new BlueDataSerializationController ();
+         mySerializer = new BlueDataSerializationController();
       }
 
       #region IPubTablePublisher Members
 
-      public virtual void addRow (IBlueRow row)
-      {
+      public virtual void addRow(IBlueRow row) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.add, 
-                                 myGetRowCollectionArray (row),
+         mySerializer.serialize(StreamOpCode.add,
+                                 myGetRowCollectionArray(row),
                                  out data);
-         myPublish (data, row.schema.tableName);
+         myPublish(data, row.schema.tableName);
       }
 
-      public virtual void deleteRow (IBlueRow row)
-      {
+      public virtual void deleteRow(IBlueRow row) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.remove, 
-                                 myGetRowCollectionArray (row),
+         mySerializer.serialize(StreamOpCode.remove,
+                                 myGetRowCollectionArray(row),
                                  out data);
-         myPublish (data, row.schema.tableName);
+         myPublish(data, row.schema.tableName);
       }
 
-      public virtual void changeRow(IBlueRow row)
-      {
+      public virtual void changeRow(IBlueRow row) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.change, 
-                                 myGetRowCollectionArray (row),
+         mySerializer.serialize(StreamOpCode.change,
+                                 myGetRowCollectionArray(row),
                                  out data);
-         myPublish (data, row.schema.tableName);
+         myPublish(data, row.schema.tableName);
       }
 
-      public virtual void createTable (IBlueRowCollection rows)
-      {
+      public virtual void createTable(IBlueRowCollection rows) {
          byte[] data;
-         mySerializer.serialize (StreamOpCode.create,
-                                 new IBlueRowCollection [] {rows},
+         mySerializer.serialize(StreamOpCode.create,
+                                 new IBlueRowCollection[] { rows },
                                  out data);
-         myPublish (data, rows.schema.tableName);
+         bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
+         myPublish(data, rows.schema.tableName);
       }
 
-      public virtual void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows)
-      {
+      public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
          byte[] data;
-         if (myIsRealChange (delRows, addRows))
-            batchChange (addRows);
-         else
-         {
-            mySerializer.serialize (StreamOpCode.multipleRemoveAdd,
-                                    new IBlueRowCollection []{delRows, addRows},
+         if (myIsRealChange(delRows, addRows))
+            batchChange(addRows);
+         else {
+            mySerializer.serialize(StreamOpCode.multipleRemoveAdd,
+                                    new IBlueRowCollection[] { delRows, addRows },
                                     out data);
-            myPublish (data, delRows.schema.tableName);
+            myPublish(data, delRows.schema.tableName);
          }
       }
 
-      public virtual void batchChange (IBlueRowCollection rows)
-      {
+      public virtual void batchChange(IBlueRowCollection rows) {
          byte[] data;
-         
-         mySerializer.serialize (StreamOpCode.multipleChange,
-                                 new IBlueRowCollection [] {rows},
+
+         mySerializer.serialize(StreamOpCode.multipleChange,
+                                 new IBlueRowCollection[] { rows },
                                  out data);
-         myPublish (data, rows.schema.tableName);
+         myPublish(data, rows.schema.tableName);
       }
 
       #endregion
 
       #region private methods
-      private void myPublish (byte[] data, string tableName)
-      {
+      private void myPublish(byte[] data, string tableName) {
+         bts.Log.Gen.info($"PubTablePublisher.myPublish table: {tableName} data: {data.Length}");
          if (data.Length > 0)
-            myPubCore.publish ("DataStreamTag", 
-                                 PubStreamPrefixer.prefix (tableName),
-                                 new MemoryStream (data, 0, data.Length, false, true));
+            myPubCore.publish("DataStreamTag",
+                                 PubStreamPrefixer.prefix(tableName),
+                                 new MemoryStream(data, 0, data.Length, false, true));
       }
 
-      private IBlueRowCollection[] myGetRowCollectionArray (IBlueRow r)
-      {
-         return new IBlueRowCollection []{
+      private IBlueRowCollection[] myGetRowCollectionArray(IBlueRow r) {
+         return new IBlueRowCollection[]{
                                             BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r})
                                          };
       }
 
-      private bool myIsRealChange (IBlueRowCollection delRows, IBlueRowCollection addRows)
-      {
-         if (delRows.Count == addRows.Count)
-         {
+      private bool myIsRealChange(IBlueRowCollection delRows, IBlueRowCollection addRows) {
+         if (delRows.Count == addRows.Count) {
             foreach (IBlueRow dRow in delRows)
-               if (!addRows.containsID ((int)dRow["id"]))
+               if (!addRows.containsID((int)dRow["id"]))
                   return false;
             return true;
-         }
-         else
+         } else
             return false;
       }
       #endregion
    }
 
-   public class IndependentTablePublisher : PubTablePublisher
-   {
+   public class IndependentTablePublisher : PubTablePublisher {
       bool myIsCreated = false;
 
-      public IndependentTablePublisher (PublisherCore pubCore) : base (pubCore)
-      {}
+      public IndependentTablePublisher(string tag, PublisherCore pubCore) : base(tag, pubCore) { }
 
-      public override void addRow (IBlueRow row)
-      {
-         myCheckCreate (row.schema);
-         base.addRow (row);
+      public override void addRow(IBlueRow row) {
+         myCheckCreate(row.schema);
+         base.addRow(row);
       }
 
-      public override void batchChange (IBlueRowCollection rows)
-      {
-         myCheckCreate (rows.schema);
-         base.batchChange (rows);
+      public override void batchChange(IBlueRowCollection rows) {
+         myCheckCreate(rows.schema);
+         base.batchChange(rows);
       }
 
-      public override void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows)
-      {
-         myCheckCreate (delRows.schema);
-         base.batchRemoveAdd (delRows, addRows);
+      public override void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
+         myCheckCreate(delRows.schema);
+         base.batchRemoveAdd(delRows, addRows);
       }
 
-      public override void changeRow (IBlueRow row)
-      {
-         myCheckCreate (row.schema);
-         base.changeRow (row);
+      public override void changeRow(IBlueRow row) {
+         myCheckCreate(row.schema);
+         base.changeRow(row);
       }
 
-      public override void deleteRow (IBlueRow row)
-      {
-         myCheckCreate (row.schema);
-         base.deleteRow (row);
+      public override void deleteRow(IBlueRow row) {
+         myCheckCreate(row.schema);
+         base.deleteRow(row);
       }
 
-      public override void createTable(IBlueRowCollection rows)
-      {
+      public override void createTable(IBlueRowCollection rows) {
          myIsCreated = true;
-         base.createTable (rows);
+         base.createTable(rows);
       }
 
 
-      private void myCheckCreate (IBlueTableSchema schema)
-      {
-         if (!myIsCreated)
-         {
+      private void myCheckCreate(IBlueTableSchema schema) {
+         if (!myIsCreated) {
             //so that the initiazliation work is done on the serializer
             byte[] placeHolder;
-            mySerializer.serialize (StreamOpCode.create, new IBlueRowCollection [] {new BlueTable (schema)}, out placeHolder);
+            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { new BlueTable(schema) }, out placeHolder);
             myIsCreated = true;
          }
       }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs

@@ -1,18 +1,16 @@
 using System;
 
-namespace BCG
-{
-	/// <summary>
-	/// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
-	/// the table name, from the config, and wraps around it. The external code that will deal with user calls for
-	/// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
-	/// serialization.
-	/// </summary>
-	public class BlueDataSerializationController : IBlueDataSerializer
-	{
+namespace BCG {
+   /// <summary>
+   /// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
+   /// the table name, from the config, and wraps around it. The external code that will deal with user calls for
+   /// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
+   /// serialization.
+   /// </summary>
+   public class BlueDataSerializationController : IBlueDataSerializer {
       private IBlueDataSerializer mySerializer;
 
-		public BlueDataSerializationController () {}
+      public BlueDataSerializationController() { }
 
       #region IBlueDataSerializer Members
 
@@ -23,43 +21,37 @@ namespace BCG
       /// <param name="op"></param>
       /// <param name="rows"></param>
       /// <param name="serializedData"></param>
-      public void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData)
-      {
-         if (op == StreamOpCode.create)
-         {
+      public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) {
+         if (op == StreamOpCode.create) {
             if (mySerializer == null)
                mySerializer = myGetSerializerForTable(rows[0].schema);
-                                                         
-            mySerializer.serialize (op, rows, out serializedData);
-         }
-         else
-         {
-            if (mySerializer != null)
-            {
-               mySerializer.serialize (op, rows, out serializedData);
-            }
-            else
-            {
-               serializedData = new byte[] {};
+
+            mySerializer.serialize(op, rows, out serializedData);
+         } else {
+            if (mySerializer != null) {
+               mySerializer.serialize(op, rows, out serializedData);
+            } else {
+               serializedData = new byte[] { };
             }
          }
       }
       #endregion
 
-      private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema)
-      {
+      private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) {
+         // TODOds
+         return new UniversalSerializer(theSchema);
          string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName);
 
          object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName);
 
-         if(serializerArgs != null)
+         if (serializerArgs != null)
             return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer;
          else
             return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer;
 
-         
+
 
       }
-      
+
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs

@@ -8,7 +8,7 @@ namespace BCG
 	public class UniversalSerializer : BlueDataSerializer
 	{
 		public UniversalSerializer (IBlueTableSchema schema)
-          : base ("BCG.PubsubUniversalDeserializer, DataStreamGateway", "BCG.ClientUniversalDeserializer")
+          : base ("BCG.PubsubUniversalDeserializer, DataStreamCore", "BCG.ClientUniversalDeserializer")
 		{
          myRowSerializer = new UniversalRowSerializer (schema);
 		}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs

@@ -7,10 +7,11 @@ using System.Text;
 namespace BCG
 {
    /// <summary>
-   /// Base class for all deserializers. 
+   /// Base class for all pubsubs deserializers. 
    /// </summary>
 	public abstract class PubsubDeserializer : IPubsubDeserializer
 	{
+      // this is where the stream state is stored in pubsub?
       protected IDictionary myState = new Hashtable ();
       protected IBlueTableSchema mySchema;
       protected string myClientDeserializerName;   //for the getState method
@@ -35,6 +36,7 @@ namespace BCG
             }
             catch (EndOfStreamException) 
             {
+               // TODO, logically break loop
                break;
             }
             if (op == StreamOpCode.create)

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs

@@ -2,24 +2,22 @@ using System;
 using System.Collections;
 using System.IO;
 
-namespace BCG
-{
-	public class PubsubUniversalDeserializer : PubsubDeserializer
-	{
-		public PubsubUniversalDeserializer() {}
+namespace BCG {
+   public class PubsubUniversalDeserializer : PubsubDeserializer {
+      public PubsubUniversalDeserializer() {
+         bts.Log.Gen.info("PubsubUniversalDeserializer.cons");
+      }
 
-      protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
-      {
-         myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myDeserializer;
-         Hashtable retVal = new Hashtable ();
-         for (int i = 0; i < noOfRows; i++)
-         {
+      protected override IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) {
+         myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
+         Hashtable retVal = new Hashtable();
+         for (int i = 0; i < noOfRows; i++) {
             int id; object row;
-            myDeserializer.deserializeRow (myState, br, op, out id, out row);
-            retVal.Add (id, row);
+            myDeserializer.deserializeRow(myState, br, op, out id, out row);
+            retVal.Add(id, row);
          }
          return retVal;
       }
 
-	}
+   }
 }

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs

@@ -1,54 +1,46 @@
 using System;
 using System.IO;
-using System.Collections;
+using System.Collections.Generic;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Keeps a collection of TagStateBanks to hold the state of each tag
    /// </summary>
-	public class PubsubServerTagHandler : IPubSubServerTagHandler
-	{
-      private IDictionary myTagStates = new Hashtable ();
+	public class PubsubServerTagHandler : IPubSubServerTagHandler {
+      private IDictionary<string, TagStateBank> _tag_state_map = new Dictionary<string, TagStateBank>();
       private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats;
          false;
-      
-      public PubsubServerTagHandler() {}
-     
+
       #region IPubSubServerTagHandler Members
 
-      public byte[] getContent (string tag)
-      {
-         TagStateBank stateBank = myTagStates[tag] as TagStateBank;
-         if (stateBank != null)
-         {
-            return stateBank.getConent ();
-         }
-         else
-         {
-            // BlueEventLog.logEvent (String.Format ("Information requested for tag {0} which does not exist now", tag), BlueEventLevel.Warning);
-            return new byte[] {};
+      public byte[] getContent(string tag) {
+         TagStateBank tag_state_bank;
+         if (_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
+            return tag_state_bank.getConent();
+         } else {
+            bts.Log.Gen.warn("Information requested for tag: {tag}, it does not exist now");
+            return new byte[] { };
          }
       }
 
-      public void update (string tag, Stream[] messageBodies)
-      {
-         if (myPublishStatistics)
-         {
+      public void update(string tag, Stream[] messageBodies) {
+         bts.Log.Gen.info($"PubsubServerTagHandler.update tag: {tag} streams: {messageBodies.Length}");
+         if (myPublishStatistics) {
             StatisticsLogger.instance.log(tag, messageBodies);
          }
-         if (!myTagStates.Contains (tag))
-         {
-            // BlueEventLog.logEvent (String.Format ("Tag {0} received for the first time and its state created", tag), BlueEventLevel.Information);
-            myTagStates.Add (tag, new TagStateBank ());
+         TagStateBank tag_state_bank;
+         if (!_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
+            bts.Log.Gen.info("tag: {tag} received for the first time creating TagStateBank for it now");
+            tag_state_bank = new TagStateBank();
+            _tag_state_map.Add(tag, tag_state_bank);
          }
-         (myTagStates[tag] as TagStateBank).update (messageBodies);
+         tag_state_bank.update(messageBodies);
       }
 
-      public void doWork() {}
+      public void doWork() { }
 
-      public string[] supportedTagTypes {get {return new string [] {"DataStreamTag"};}}
+      public string[] supportedTagTypes { get { return new string[] { "DataStreamTag" }; } }
 
       public void register(InternalPublishDelegate callback) { }
 

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs

@@ -1,26 +1,26 @@
 using System;
-using System.Collections;
+using System.Collections.Generic;
 using System.IO;
 
-namespace BCG
-{
-   public class TagStateBank
-   {
-      //private IDictionary myState = new Hashtable ();
-      private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController ();
-		
-      public TagStateBank () {}
+namespace BCG {
+   public class TagStateBank {
+      private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController();
 
-      public byte[] getConent ()
-      {
-         return myDeserializationController.getState ();
+      public TagStateBank() { }
+
+      public TagStateBank(string tag) {
+         bts.Log.Gen.info("TagStateBank created for tag: {tag}");
       }
 
-      public void update (Stream[] streams)
-      {
+      public byte[] getConent() {
+         return myDeserializationController.getState();
+      }
+
+      public void update(Stream[] streams) {
+         bts.Log.Gen.info($"TagStateBank.update streams: {streams.Length}");
          byte[] ops;
          Array rowCollection;
-         myDeserializationController.deserialize (streams, out ops, out rowCollection);
+         myDeserializationController.deserialize(streams, out ops, out rowCollection);
       }
    }
 }

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs

@@ -9,7 +9,7 @@ namespace BCG
    public class CompressionSerializer : IBlueDataSerializer
    {
       private IBlueRowSerializer myRowSerializer;
-      private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamGateway", myClientDeserializerName = "BCG.CompressionClientDeserializer";
+      private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamCore", myClientDeserializerName = "BCG.CompressionClientDeserializer";
       
       public CompressionSerializer (IBlueTableSchema schema)
       {

src/DataStreamSuite/DataStreamCore/Messaging/Test/CompressionSerializationTest.cs

@@ -0,0 +1,236 @@
+// using System;
+// using System.IO;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+
+//    /// <summary>
+//    /// Tests the deserialization happening within the pubsub.
+//    /// </summary>
+//    [TestGroup]
+//    public class CompressionSerializationTest
+//    {
+//       protected IBlueTableSchema mySchema;
+//       protected BlueTable myTable;
+//       protected IBlueDataSerializer mySer;
+//       protected IPubsubDeserializer myPubsubDeser;
+//       protected IClientDeserializer myClientDeser;
+
+
+//       [Test]
+//       public static void test_compression()
+//       {
+//          int n_len = 10000000;
+//          for (int t = 0; t < 10; t++)
+//          {
+//             Random rnd = new Random(DateTime.Now.Millisecond);
+//             byte[] data = new byte[n_len];
+//             rnd.NextBytes(data);
+
+//             BCG.Utilities.General.HighPerformanceTimer timer = new Utilities.General.HighPerformanceTimer();
+//             timer.start();
+//             byte[] compr = UtilityMethods.compress(new MemoryStream(data));
+//             timer.stop();
+//             Console.WriteLine("Time to compress {0} bytes {1} msecs ratio {2}", n_len, timer.lastIntervalTime, (1.0 * data.Length)/(1.0 * compr.Length));
+
+//             timer.start();
+//             Stream decompr_str = UtilityMethods.decompress(compr);
+//             timer.stop();            
+//             Console.WriteLine("Time to de-compress {0} bytes {1} msecs", n_len, timer.lastIntervalTime);
+
+//             decompr_str.Position = 0;
+//             Assert.AreEqual(data.Length, (int)decompr_str.Length);
+//             for (int i = 0; i < data.Length; i++)
+//                Assert.AreEqual(data[i], decompr_str.ReadByte());
+//          }
+//       }
+
+//       public CompressionSerializationTest()
+//       {
+//          mySchema = new BlueTableSchema(myGetTableName(),
+//                                           new ColumnInfo("id", 0, typeof(int)),
+//                                           new ColumnInfo("i1", 1, typeof(int)),
+//                                           new ColumnInfo("d1", 2, typeof(double)),
+//                                           new ColumnInfo("dt", 3, typeof(DateTime)),
+//                                           new ColumnInfo("x", 4, typeof(byte))
+//             );
+//          mySer = new CompressionSerializer(mySchema);
+//          myClientDeser = new ClientDeserializationController();
+//          myPubsubDeser = new PubsubDeserializationController();
+//       }
+
+//       protected virtual string myGetTableName()
+//       {
+//          return "testUniversal";
+//       }
+
+//       /// <summary>
+//       /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
+//       /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer. 
+//       /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
+//       /// through the clientDeserializer to get all the rows.
+//       /// </summary>
+//       [Test]
+//       public void testEntireProcess()
+//       {
+//          myTable = new BlueTable(mySchema);
+//          byte[] data;
+//          int id;
+//          //create
+//          for (id = 0; id < 100; id++)
+//             myTable.addRow(myGetRow(id));
+
+//          byte[] ops;
+//          Array rows;
+//          IBlueRowCollection del, add;
+//          IBlueRowCollection[] receivedData;
+//          IBlueRow r;
+
+//          //create
+//          mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
+//          myWrite(StreamOpCode.create, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.create, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+
+//          BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //single add
+//          r = myGetRow(id);
+//          BlueTable temp = new BlueTable(mySchema);
+//          myTable.addRow(r);
+//          temp.addRow(r);
+//          mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
+//          myWrite(StreamOpCode.add, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.add, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //single delete
+//          r = myTable[0];   //delete row with id 0
+//          temp = new BlueTable(mySchema);
+//          myTable.deleteRow(0);
+//          temp.addRow(r);
+//          mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
+//          myWrite(StreamOpCode.remove, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.remove, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //batch deletes and adds
+//          //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
+//          BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
+//          for (int i = 1; i < 25; i++)
+//          {
+//             delRows.addRow(myTable[i]);
+//             myTable.deleteRow(i);
+//          }
+//          for (id = 101; id < 125; id++)
+//          {
+//             r = myGetRow(id);
+//             addRows.addRow(r);
+//             myTable.addRow(r);
+//          }
+//          for (int i = 25; i < 125; i++)
+//             Assert.AreEqual(true, myTable.containsID(i));
+//          Assert.AreEqual(100, myTable.Count);
+
+//          mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
+//          myWrite(StreamOpCode.multipleRemoveAdd, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(2, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+//          BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //pure changes only
+//          //change the first 25, 25-49
+//          BlueTable changedRows = new BlueTable(mySchema);
+//          for (int i = 25; i < 50; i++)
+//          {
+//             MutableBlueRow mr = new MutableBlueRow(myTable[i]);
+//             mr.setValue(4, Byte.MaxValue);
+//             myTable.changeRow(mr);
+//             changedRows.addRow(mr);
+//          }
+//          mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
+//          myWrite(StreamOpCode.multipleChange, data);
+//          myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//          //final test
+//          data = myPubsubDeser.getState();
+//          new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+//          Assert.AreEqual(1, ops.Length);
+//          Assert.AreEqual(StreamOpCode.create, ops[0]);
+//          Assert.AreEqual(1, rows.Length);
+//          receivedData = rows.GetValue(0) as IBlueRowCollection[];
+//          Assert.AreEqual(1, receivedData.Length);
+//          BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+//          Assert.AreEqual(0, del.Count);
+//          Assert.AreEqual(0, add.Count);
+
+//       }
+
+//       private IBlueRow myGetRow(int id)
+//       {
+//          return new BlueRow(mySchema,                        // the day pp fixed this test. 
+//                               id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
+//       }
+
+//       private void myWrite(byte op, byte[] data)
+//       {
+//          string s = "";
+//          if (op == StreamOpCode.add)
+//             s = "add one row";
+//          else if (op == StreamOpCode.change)
+//             s = "change one row";
+//          else if (op == StreamOpCode.create)
+//             s = "create";
+//          else if (op == StreamOpCode.multipleChange)
+//             s = "multiple change";
+//          else if (op == StreamOpCode.multipleRemoveAdd)
+//             s = "multiple add remove";
+//          else if (op == StreamOpCode.remove)
+//             s = "remove one row";
+//          Console.WriteLine("op = {0}, size = {1}", s, data.Length);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingRowSerializationTest.cs

@@ -0,0 +1,100 @@
+// using System;
+// using System.IO;
+// using System.Collections;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+//    [TestGroup]
+// 	public class FixedChangingRowSerializationTest
+// 	{
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
+//                                                                new ColumnInfo ("i", 0, typeof(int)),
+//                                                                new ColumnInfo ("b", 1, typeof(bool)),
+//                                                                new ColumnInfo ("d", 2, typeof(double)),
+//                                                                new ColumnInfo ("id", 3, typeof(int)),
+//                                                                new ColumnInfo ("dt", 4, typeof(DateTime)),
+//                                                                new ColumnInfo ("bt", 5, typeof(byte)));
+//       private BlueTable myState;
+//       private Random myRnd = new Random (DateTime.Now.Millisecond);
+//       private IBlueRowSerializer mySerializer;
+//       private IClientRowDeserializer myRowDeserializer;
+
+// 		public FixedChangingRowSerializationTest ()
+// 		{
+//          myState = new BlueTable (mySchema);  
+//          mySerializer = new FixedChangingRowSerializer (mySchema, "bt");
+//          myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, new string[] {"bt"});
+// 		}
+
+//       [Test]
+//       public void addTest ()
+//       {
+//          for (int i = 0; i < 1000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow r = myCreateRow (i);
+//             mySerializer.serializeRow (r, StreamOpCode.add, bw);   
+            
+//             BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
+//             Assert.AreEqual (r["i"], br.ReadInt32());
+//             Assert.AreEqual (r["b"], br.ReadBoolean ());
+//             Assert.AreEqual (r["d"], br.ReadDouble ());
+//             Assert.AreEqual (r["id"], br.ReadInt32 ());
+//             Assert.AreEqual (((DateTime)r["dt"]).Ticks, br.ReadInt64());
+//             Assert.AreEqual (r["bt"], br.ReadByte ());
+//          }
+//       }
+
+//       [Test]
+//       public void removeTest ()
+//       {
+//          for (int i = 0; i < 1000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow r = myCreateRow (i);
+//             mySerializer.serializeRow (r, StreamOpCode.remove, bw);   
+            
+//             BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
+//             Assert.AreEqual (r["id"], br.ReadInt32());
+//          }
+//       }
+
+//       [Test]
+//       public void changeTest ()
+//       {
+//          for (int i = 0; i < 1000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow r = myCreateRow (i);
+//             mySerializer.serializeRow (r, StreamOpCode.change, bw);   
+            
+//             BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
+//             Assert.AreEqual (r["id"], br.ReadInt32());
+//             Assert.AreEqual (r["bt"], br.ReadByte());
+//             bool isEnd = false;
+//             try
+//             {
+//                br.ReadByte ();
+//             }
+//             catch (EndOfStreamException)
+//             {
+//                isEnd = true;
+//             }
+//             Assert.That (isEnd);
+//          }
+//       }
+      
+//       private IBlueRow myCreateRow (int id)
+//       {
+//          return new BlueRow (mySchema,
+//                               myRnd.Next (1000),
+//                               myRnd.Next (1000) % 2 == 0,
+//                               myRnd.NextDouble (),
+//                               id,
+//                               DateTime.Now,
+//                               Convert.ToByte (10));
+//       }
+
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingSerializationTest.cs

@@ -0,0 +1,40 @@
+// using System;
+
+// namespace BCG
+// {
+
+//    public class TestFixedChangingSerializer : FixedChangingSerializer
+//    {
+//       public TestFixedChangingSerializer (IBlueTableSchema schema)
+//          : base (schema, 
+//                   new string[] {"x"}, 
+//                   "BCG.PubsubTestFixedChangingDeserializer",
+//                   "BCG.ClientTestFixedChangingDeserializer"
+//                 )
+//       {}                                              
+//    }
+
+//    public class PubsubTestFixedChangingDeserializer : PubsubFixedChangingDeserializer
+//    {
+//       public PubsubTestFixedChangingDeserializer () : base (new string[] {"x"})
+//       {}
+//    }
+
+//    public class ClientTestFixedChangingDeserializer : ClientFixedChangingDeserializer
+//    {
+//       public ClientTestFixedChangingDeserializer () : base (new string[] {"x"})
+//       {}
+//    }
+
+
+// 	public class FixedChangingSerializationTest : UniversalSerializationTest
+// 	{
+// 		public FixedChangingSerializationTest () {}
+
+//       protected override string myGetTableName()
+//       {
+//          return "testFixedChanging";
+//       }
+
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/MarketOptionTheoSerializationTest.cs

@@ -0,0 +1,73 @@
+// using System;
+// using System.Collections;
+// using System.IO;
+// using NUnit.Framework;
+// using BCG.Utilities.General;
+
+// namespace BCG
+// {
+//    [TestFixture]
+// 	public class MarketOptionTheoSerializationTest
+// 	{
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("marketOptionTheo",
+//                                                                   new ColumnInfo ("arrayIndex", 0, typeof(byte)),
+//                                                                   new ColumnInfo ("cell", 1, typeof(byte)),
+//                                                                   new ColumnInfo ("expiration", 2, typeof(DateTime)),
+//                                                                   new ColumnInfo ("id", 3, typeof(int)),
+//                                                                   new ColumnInfo ("value", 4, typeof(double))
+//                                                                );
+                                          
+// 		public MarketOptionTheoSerializationTest () {}
+
+//       [Test]
+//       public void serializationPerformanceTest ()
+//       {
+//          MarketOptionTheoSerializer ser = new MarketOptionTheoSerializer (mySchema);
+//          IClientDeserializer deser = new MarketOptionTheoClientDeserializer ();
+//          BlueTable bt = new BlueTable (mySchema);
+//          int count = 3000;
+//          byte[] ops;
+//          Array rows;
+//          HighPerformanceTimer timer = new HighPerformanceTimer ();
+//          byte[] serData = null;
+
+//          for (int i = 0; i < count; i++)
+//             bt.addRow (myGetRow (i));
+
+         
+//          ///serialization
+//          //create
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out serData);
+//          deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
+//          ArrayList times = new ArrayList ();
+//          for (int i = 0; i < 25; i++)
+//          {
+//             timer.start ();
+//             ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {bt, bt}, out serData);
+//             timer.stop ();
+//             times.Add (timer.lastIntervalTime);
+//             Console.WriteLine (String.Format ("serialization of {0} rows, time = {1}, size = {2}", count, timer.lastIntervalTime, serData.Length));
+//          }
+//          times.Sort ();
+//          Console.WriteLine ("Median time = {0}", times[times.Count/2]);
+//          //deserialization
+//          times.Clear ();
+//          for (int i = 0; i < 25; i++)
+//          {
+//             timer.start ();
+//             deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
+//             timer.stop();
+//             times.Add (timer.lastIntervalTime);
+//             Console.WriteLine (String.Format ("deserialization of {0} rows, time = {1}", count, timer.lastIntervalTime));
+//          }
+//          times.Sort ();
+//          Console.WriteLine ("Median time = {0}", times[times.Count/2]);
+//       }
+
+//       private IBlueRow myGetRow (int id)
+//       {
+//          return new BlueRow (mySchema,
+//                              Convert.ToByte (2), Convert.ToByte (4), DateTime.Now, id, 10.324555);
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/Performance.cs

@@ -0,0 +1,46 @@
+// using System;
+// using NUnit.Framework;
+// using BCG.Utilities.General;
+
+// namespace BCG
+// {
+//    [TestFixture]
+// 	public class Performance
+// 	{
+// 		public Performance()
+// 		{
+// 		}
+
+//       [Test]
+//       public void testContructionTime ()
+//       {
+//          double totalTime;
+//          string deserName = "BCG.ClientUniversalDeserializer";
+//          HighPerformanceTimer timer = new HighPerformanceTimer ();
+//          IClientDeserializer cd;
+//          for (int i = 0; i < 100; i++)
+//          {
+//             timer.start ();
+//             cd = Activator.CreateInstance (Type.GetType (deserName)) as IClientDeserializer;
+//             timer.stop ();
+//             //Console.Write (timer.lastIntervalTime.ToString() + " ");
+//          }
+//          Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
+//          totalTime = timer.totalMilliseconds;
+
+//          timer = new HighPerformanceTimer ();
+//          for (int i = 0; i < 100; i++)
+//          {
+//             timer.start ();
+//             cd = new ClientUniversalDeserializer ();
+//             timer.stop ();
+//             //Console.Write (timer.lastIntervalTime.ToString () + " ");
+//          }
+//          Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
+//          Console.WriteLine ("Ratio = " + totalTime/timer.totalMilliseconds);
+//       }
+
+
+
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/TableAccumulatorTest.cs

@@ -0,0 +1,275 @@
+// using System;
+// using System.IO;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+// 	[TestFixture]
+// 	public class TableAccumulatorTest
+// 	{
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("test",
+//                                                                new ColumnInfo ("id", 0, typeof(int)),
+//                                                                new ColumnInfo ("x", 1, typeof(int)));
+//       private SubscriberCore mySubCore = new SubscriberCore ();
+//       private IBlueRowCollection myNr, myAr, myDr;
+		
+//       public TableAccumulatorTest() {}
+
+//       [SetUp]
+//       public void tearDown () 
+//       {
+//          myNr = myAr = myDr = null;
+//       }
+
+//       #region create
+//       [Test]
+//       public void testCreate ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //testing the data
+//          IBlueRowCollection dr, ar;
+//          //new rows
+//          BlueDataUtility.computeDifferences (bt, myNr, out dr, out ar);
+//          Assert.AreEqual (0, dr.Count);
+//          Assert.AreEqual (0, ar.Count);
+//          //added rows
+//          BlueDataUtility.computeDifferences (bt, myAr, out dr, out ar);
+//          Assert.AreEqual (0, dr.Count);
+//          Assert.AreEqual (0, ar.Count);
+//          //deleted rows;
+//          Assert.AreEqual (0, myDr.Count);
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region single add, delete and change
+//       [Test]
+//       public void testAddRemoveChange ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //add
+//          //add a row with id 100
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 100, 200));
+//          ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //testing the add data
+//          Assert.AreEqual (101, myNr.Count);
+//          for (int i = 0; i < 101; i++)
+//             myNr.containsID (i);
+//          Assert.AreEqual (0, myDr.Count);
+//          Assert.AreEqual (1, myAr.Count);
+//          Assert.That (myAr.containsID (100));
+
+//          //remove
+//          //remove the row with id 0
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 0, 0));
+//          ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //testing remove data
+//          Assert.AreEqual (100, myNr.Count);
+//          for (int i = 0; i < 101; i++)
+//          {
+//             Assert.That ((i == 0 && !myNr.containsID (i)) || (i != 0 && myNr.containsID (i)));
+//          }
+//          Assert.AreEqual (0, myAr.Count);
+//          Assert.AreEqual (1, myDr.Count);
+//          Assert.That (myDr.containsID (0));
+         
+//          //change
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 100, -100));
+//          ser.serialize (StreamOpCode.change, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //testing changed data
+//          Assert.AreEqual (100, myNr.Count);
+//          Assert.AreEqual (1, myAr.Count);
+//          Assert.AreEqual (100, BlueDataUtility.extractRow (myAr, 0)["id"]);
+//          Assert.AreEqual (-100, BlueDataUtility.extractRow (myAr, 0)["x"]);
+//          Assert.AreEqual (1, myDr.Count);
+//          Assert.AreEqual (100, BlueDataUtility.extractRow (myDr, 0)["id"]);
+//          Assert.AreEqual (200, BlueDataUtility.extractRow (myDr, 0)["x"]);
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region multiple change
+//       public void testMultipleChange ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //partial rows changed
+//          bt.clear ();
+//          for (int i = 0; i < 50; i++)
+//             bt.addRow (new BlueRow (mySchema, i, -2*i));
+//          ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //checking received data
+//          Assert.AreEqual (100, myNr.Count);
+//          foreach (IBlueRow r in myNr)
+//          {
+//             if ((int)r["id"] < 50)
+//                Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//             else
+//                Assert.AreEqual ((int)r["id"] * 2, r["x"]);
+//          }
+//          Assert.AreEqual (50, myAr.Count);
+//          foreach (IBlueRow r in myAr)
+//             Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//          Assert.AreEqual (50, myDr.Count);
+//          foreach (IBlueRow r in myDr)
+//             Assert.AreEqual ((int)r["id"] * 2, r["x"]);
+
+//          //full rows changed
+//          bt.clear ();
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, -2*i));
+//          ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //checking received data
+//          Assert.AreEqual (100, myNr.Count);
+//          foreach (IBlueRow r in myNr)
+//          {
+//             Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//          }
+//          Assert.AreEqual (100, myAr.Count);
+//          foreach (IBlueRow r in myAr)
+//             Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//          Assert.AreEqual (100, myDr.Count);
+//          foreach (IBlueRow r in myDr)
+//          {
+//             if ((int)r["id"] < 50)
+//                Assert.AreEqual ((int)r["id"] * -2, r["x"]);
+//             else
+//                Assert.AreEqual ((int)r["id"] * 2, r["x"]);
+//          }
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region multiple deletes/adds
+//       public void testMultipleDeletesAdds ()
+//       {
+         
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          BlueTable at = new BlueTable (mySchema),
+//                      dt = new BlueTable (mySchema);
+//          for (int i = 0; i < 50; i++)
+//             dt.addRow (new BlueRow (mySchema, i, 2*i));
+//          for (int i = 100; i < 150; i++)
+//             at.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dt, at}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+
+//          //checking state
+//          Assert.AreEqual (100, myNr.Count);
+//          for (int i = 50; i < 150; i++)
+//             Assert.That (myNr.containsID (i));
+//          Assert.AreEqual (50, myDr.Count);
+//          for (int i = 0; i < 50; i++)
+//             Assert.That (myDr.containsID (i));
+//          Assert.AreEqual (50, myAr.Count);
+//          for (int i = 100; i < 150; i++)
+//             Assert.That (myAr.containsID (i));
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+//       }
+//       #endregion
+
+//       #region multiple ops in one stream
+//       [Test]
+//       public void testMultipleOps ()
+//       {
+//          TableAccumulator ta = new TableAccumulator ("test", mySubCore);
+//          ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
+//          IBlueDataSerializer ser = new BlueDataSerializationController ();
+//          byte[] data;
+//          MemoryStream ms = new MemoryStream ();
+//          BlueTable bt = new BlueTable (mySchema);
+//          for (int i = 0; i < 100; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
+//          ta.update (new Stream[] {new MemoryStream (data)});
+//          //ms.Write (data, 0, data.Length);
+//          //changing the first 5 rows
+//          bt.clear ();
+//          for (int i = 0; i < 5; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 1000));
+//          ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          //delete id 5
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 5, 10));
+//          ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          //add id 100
+//          bt.clear ();
+//          bt.addRow (new BlueRow (mySchema, 100, 200));
+//          ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          //delete 4 and add 5
+//          bt.clear ();
+//          BlueTable dels = new BlueTable (bt.schema);
+//          for (int i = 90; i < 94; i++)
+//             dels.addRow (new BlueRow (mySchema, i, i*2));
+         
+//          for (int i = 101; i < 106; i++)
+//             bt.addRow (new BlueRow (mySchema, i, 2*i));
+//          ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dels, bt}, out data);
+//          ms.Write (data, 0, data.Length);
+//          ms.Position = 0;
+//          ta.update (new Stream[] {ms});
+         
+//          Assert.AreEqual (101, myNr.Count);
+//          Assert.AreEqual (10, myDr.Count);
+//          Assert.AreEqual (11, myAr.Count);
+//          ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
+         
+//       }
+//       #endregion
+
+//       private void myHandleUpdate (IBlueRowCollection nr, IBlueRowCollection ar, IBlueRowCollection dr)
+//       {
+//          myNr = nr;
+//          myAr = ar;
+//          myDr = dr;
+//       }
+
+      
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Test/UniversalRowSerializerTest.cs

@@ -0,0 +1,58 @@
+// using System;
+// using System.IO;
+// using NUnit.Framework;
+
+// namespace BCG
+// {
+//    [TestFixture]
+// 	public class UniversalRowSerializerTest
+// 	{
+//       private IBlueRowSerializer myRowSerializer;
+//       private IClientRowDeserializer myRowDeserializer;
+//       private Random myRnd = new Random (DateTime.Now.Millisecond);
+//       private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
+//                                                                new ColumnInfo ("id", 0, typeof(int)),
+//                                                                new ColumnInfo ("int1", 1, typeof(int)),
+//                                                                new ColumnInfo ("int2", 2, typeof(int)),
+//                                                                new ColumnInfo ("double1", 3, typeof(double)),
+//                                                                new ColumnInfo ("double2", 4, typeof(double)),
+//                                                                new ColumnInfo ("byte1", 5, typeof(byte)),
+//                                                                new ColumnInfo ("string1", 6, typeof(string)),
+//                                                                new ColumnInfo ("string2", 7, typeof(string)));
+// 		public UniversalRowSerializerTest () {}
+
+//       [Test]
+//       public void testSerialiazationCycle ()
+//       {
+//          myRowSerializer = new UniversalRowSerializer (mySchema);
+//          myRowDeserializer = new ClientUniversalRowDeserializer (mySchema);
+//          for (int i = 0; i < 10000; i++)
+//          {
+//             BinaryWriter bw = new BinaryWriter (new MemoryStream ());
+//             IBlueRow ir = myGetRandomRow ();
+//             myRowSerializer.serializeRow (ir, StreamOpCode.add, bw);
+
+//             IBlueRow or;
+//             (bw.BaseStream as MemoryStream).Position = 0;
+//             BinaryReader br = new BinaryReader (bw.BaseStream);
+//             myRowDeserializer.deserializeRow (null, br, StreamOpCode.add, out or);
+//             Assert.That (BlueDataUtility.computeRowEquality (ir, or));
+//          }
+//       }
+
+//       private IBlueRow myGetRandomRow ()
+//       {
+//          return new BlueRow (mySchema,
+//                                     myRnd.Next (1000),
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.Next (1000) as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object: myRnd.Next (1000) as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : Convert.ToByte (76) as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "abcdefgh" as object,
+//                                     myRnd.Next (5)% 3 == 0 ? DBNull.Value as object :  "1234567890" as object
+//                                  ); 
+                                    
+//       }
+// 	}
+// }

src/DataStreamSuite/DataStreamCore/Messaging/stream_serialization.txt

@@ -0,0 +1,13 @@
+PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
+PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd)
+PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[]
+BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now)
+
+IBlueDataSerializer has just one function
+void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
+the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add
+(BlueDataSerializer.serialize's comment has the serialization layout)
+
+For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams)
+For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer
+

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj

@@ -3,6 +3,9 @@
   <ItemGroup>
     <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
     <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\pubsub\pubsub.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
   </ItemGroup>
 
   <PropertyGroup>

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -1,16 +1,35 @@
-using bts.utils;
+using System;
 
-using mbts.config;
+using BCG;
 
-using System;
-using bts;
-using mbts;
+bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
+mbts.config.Configuration.initialize();
 
-BTSNativeMethods.provide_command_line(args, args.Length);
-// Must initialize config before using log system
-Console.WriteLine("Initializing config...");
-Configuration.initialize();
-for (int i = 0; i < 10; i++)
-{
-   bts.Log.Gen.info($"Starting DataStreamPublisher...log: {i}");
+bts.Log.Gen.info($"Starting DSPublisher");
+Console.WriteLine("Starting DSPublisher...");
+// SimpleTCPClient.start("127.0.0.1", 12345);
+// Task.Delay(1000000000).Wait();
+
+//TestPubliser.publish();
+//TestPubliser.subscribe();
+// TestPublisher ts = new TestPublisher(false);
+// ts.keep_publishing(TimeSpan.FromSeconds(10));
+
+
+_subscribe_to_stream();
+
+void _subscribe_to_stream() {
+   BCG.ViewStream vs = new TableStream("literal");
+   vs.onChange += on_change;
 }
+
+while (true) {
+   System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
+   ViewStreamDatabase.instance.doWork();
+}
+
+
+void on_change(object obj) {
+   DataPacket dp = obj as DataPacket;
+   dp.newRows.print_table((col) => { return 10; });
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj

@@ -1,5 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\pubsub\pubsub.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <!-- <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> -->
+  </ItemGroup>
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net7.0-windows</TargetFramework>

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -9,9 +9,11 @@ using mbts;
 using mbts.config;
 using bts.utils;
 
-static class Program {
+static class Program
+{
    [STAThread]
-   static void Main(string[] args) {
+   static void Main(string[] args)
+   {
       BTSNativeMethods.provide_command_line(args, args.Length);
       Configuration.initialize();
       bts.Log.Gen.info($"Starting DSQA");
@@ -21,71 +23,10 @@ static class Program {
       // TestPubliser.subscribe();
       Console.WriteLine("Launching Main Window");
       MainForm mainForm = new MainForm();
-      if (args.Length > 0) {
+      if (args.Length > 0)
+      {
          mainForm.loadFile(args[0]);
       }
       Application.Run(mainForm);
    }
-}
-
-class TestPubItem : IFormatter, IPubItem
-{
-   public DateTime send_time = DateTime.Now;
-   public object Deserialize(Stream str)
-   {
-      BinaryReader br = new BinaryReader(str);
-      var d = br.ReadDouble();
-      TestPubItem rv = new TestPubItem();
-      rv.send_time = DateTime.FromOADate(d);
-      return rv;
-   }
-   public void Serialize(Stream str, Object obj)
-   {
-      TestPubItem tpi = obj as TestPubItem;
-      BinaryWriter bw = new BinaryWriter(str);
-      bw.Write(tpi.send_time.ToOADate());
-   }
-   public System.Runtime.Serialization.SerializationBinder? Binder { get; set; }
-
-   public System.Runtime.Serialization.StreamingContext Context { get; set; }
-
-   public System.Runtime.Serialization.ISurrogateSelector? SurrogateSelector { get; set; }
-
-   public string getTag()
-   {
-      return "Test_Tag";
-   }
-}
-
-class TestPubliser
-{
-   public static void publish()
-   {
-      int n_pubs = 1000;
-      Publisher<TestPubItem> publisher = new Publisher<TestPubItem>();
-      for (int i = 0; i < n_pubs; i++)
-      {
-         TestPubItem pi = new TestPubItem();
-         pi.send_time = DateTime.Now;
-         bts.Log.Gen.info($"DSQA publishing tag: {pi.getTag()}");
-         publisher.publish(pi);
-         System.Threading.Thread.Sleep(100);
-         MasterWorker.getInstance(false).doWork();
-      }
-   }
-   public static void subscribe()
-   {
-      ManagedSubscriber ms = new ManagedSubscriber();
-      ms.subscribe("Test_Tag", new Handler(on_data));
-      while (true)
-      {
-         ms.doWork();
-      }
-   }
-   public static void on_data(IPubItem pi)
-   {
-      Console.WriteLine("Received data");
-   }
-}
-
-
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -21,19 +21,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_core", "..\client\spa
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_api", "..\client\spark_api\spark_api.csproj", "{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlueRegistry", "BlueRegistry\BlueRegistry.csproj", "{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "BaseUtilities\BaseUtilities.csproj", "{A9EDEEBD-A03A-4281-8FAF-2419D3E7009C}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GuiUtilities", "GuiUtilities\GuiUtilities.csproj", "{CADBA1B3-0837-44DD-85BB-70351AD3A028}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "ArithmeticExpressions\ArithmeticExpressions.csproj", "{EFE6AD17-2564-43A1-B0E5-A2A8DA173002}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "DataStreamExpressions\DataStreamExpressions.csproj", "{D46A152D-AD29-4BEF-980B-5896694E312F}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "ColumnMapExpressions\ColumnMapExpressions.csproj", "{A844AB94-195B-4A7F-89CD-9A335985328E}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "ExpressionCore\ExpressionCore.csproj", "{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStreamTests\DataStreamTests.csproj", "{66AB10D3-155F-4663-B1BC-D0818E17B87B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -80,6 +68,10 @@ Global
 		{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Debug|Any CPU.Build.0 = Debug|x64
 		{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.ActiveCfg = Release|x64
 		{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.Build.0 = Release|x64
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="NUnit" Version="3.13.3" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
+    <PackageReference Include="NUnit.Analyzers" Version="3.5.0" />
+    <PackageReference Include="coverlet.collector" Version="3.1.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\pubsub\pubsub.csproj" />
+    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+  </ItemGroup>
+
+
+</Project>

src/DataStreamSuite/DataStreamTests/StreamSerialization/UniversalSerializationTest.cs

@@ -0,0 +1,195 @@
+using System;
+using System.IO;
+
+using NUnit.Framework;
+
+namespace BCG {
+
+   /// <summary>
+   /// Tests the deserialization happening within the pubsub.
+   /// </summary>
+   public class UniversalSerializationTest {
+      protected IBlueTableSchema mySchema;
+      protected BlueTable myTable;
+      protected IBlueDataSerializer mySer = new BlueDataSerializationController();
+      protected IPubsubDeserializer myPubsubDeser = new PubsubDeserializationController();
+      protected IClientDeserializer myClientDeser = new ClientDeserializationController();
+
+      public UniversalSerializationTest() {
+         mySchema = new BlueTableSchema(myGetTableName(),
+                                          new ColumnInfo("id", 0, typeof(int)),
+                                          new ColumnInfo("i1", 1, typeof(int)),
+                                          new ColumnInfo("d1", 2, typeof(double)),
+                                          new ColumnInfo("dt", 3, typeof(DateTime)),
+                                          new ColumnInfo("x", 4, typeof(byte))
+            );
+      }
+
+      protected virtual string myGetTableName() {
+         return "testUniversal";
+      }
+
+      /// <summary>
+      /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
+      /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer. 
+      /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
+      /// through the clientDeserializer to get all the rows.
+      /// 
+      /// </summary>
+      [Test]
+      public void testEntireProcess() {
+         myTable = new BlueTable(mySchema);
+         byte[] data;
+         int id;
+         //create
+         for (id = 0; id < 100; id++)
+            myTable.addRow(myGetRow(id));
+
+         byte[] ops;
+         Array rows;
+         IBlueRowCollection del, add;
+         IBlueRowCollection[] receivedData;
+         IBlueRow r;
+
+         //create
+         mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
+         myWrite(StreamOpCode.create, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.create, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+
+         BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //single add
+         r = myGetRow(id);
+         BlueTable temp = new BlueTable(mySchema);
+         myTable.addRow(r);
+         temp.addRow(r);
+         mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
+         myWrite(StreamOpCode.add, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.add, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //single delete
+         r = myTable[0];   //delete row with id 0
+         temp = new BlueTable(mySchema);
+         myTable.deleteRow(0);
+         temp.addRow(r);
+         mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
+         myWrite(StreamOpCode.remove, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.remove, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //batch deletes and adds
+         //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
+         BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
+         for (int i = 1; i < 25; i++) {
+            delRows.addRow(myTable[i]);
+            myTable.deleteRow(i);
+         }
+         for (id = 101; id < 125; id++) {
+            r = myGetRow(id);
+            addRows.addRow(r);
+            myTable.addRow(r);
+         }
+         for (int i = 25; i < 125; i++)
+            Assert.AreEqual(true, myTable.containsID(i));
+         Assert.AreEqual(100, myTable.Count);
+
+         mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
+         myWrite(StreamOpCode.multipleRemoveAdd, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(2, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+         BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //pure changes only
+         //change the first 25, 25-49
+         BlueTable changedRows = new BlueTable(mySchema);
+         for (int i = 25; i < 50; i++) {
+            MutableBlueRow mr = new MutableBlueRow(myTable[i]);
+            mr.setValue(4, Byte.MaxValue);
+            myTable.changeRow(mr);
+            changedRows.addRow(mr);
+         }
+         mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
+         myWrite(StreamOpCode.multipleChange, data);
+         myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+         //final test
+         data = myPubsubDeser.getState();
+         new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
+         Assert.AreEqual(1, ops.Length);
+         Assert.AreEqual(StreamOpCode.create, ops[0]);
+         Assert.AreEqual(1, rows.Length);
+         receivedData = rows.GetValue(0) as IBlueRowCollection[];
+         Assert.AreEqual(1, receivedData.Length);
+         BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
+         Assert.AreEqual(0, del.Count);
+         Assert.AreEqual(0, add.Count);
+
+      }
+
+      private IBlueRow myGetRow(int id) {
+         return new BlueRow(mySchema,                        // the day pp fixed this test. 
+                              id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
+      }
+
+      private void myWrite(byte op, byte[] data) {
+         string s = "";
+         if (op == StreamOpCode.add)
+            s = "add one row";
+         else if (op == StreamOpCode.change)
+            s = "change one row";
+         else if (op == StreamOpCode.create)
+            s = "create";
+         else if (op == StreamOpCode.multipleChange)
+            s = "multiple change";
+         else if (op == StreamOpCode.multipleRemoveAdd)
+            s = "multiple add remove";
+         else if (op == StreamOpCode.remove)
+            s = "remove one row";
+         Console.WriteLine("yes, here , op = {0}, size = {1}", s, data.Length);
+      }
+   }
+}

src/DataStreamSuite/DataStreamTests/Usings.cs

@@ -0,0 +1 @@
+global using NUnit.Framework;

src/DataStreamSuite/PubSub/AllPublishedTagState.cs

@@ -4,15 +4,13 @@ using System.Collections;
 using System.Collections.Generic;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Holds the state for a single tag. If the tagType has an
    /// IPubSubServerTagHandler, it is updated and the state stored
    /// there. Otherwise, this class saves the last published data.
    /// </summary>
-   internal class SingleTagHandler
-   {
+   internal class SingleTagHandler {
       private IPubSubServerTagHandler myHandler = null;
       private string myTagType, myTag;
       /// <summary>
@@ -20,9 +18,7 @@ namespace BCG
       /// </summary>
       private byte[] myData = null;
 
-      public SingleTagHandler(string tag, string tagType)
-      {
-         
+      public SingleTagHandler(string tag, string tagType) {
          myTagType = tagType;
          myTag = tag;
          myHandler = TagType2HandlerMap.getHandler(tagType, false);
@@ -30,23 +26,18 @@ namespace BCG
 
       public string tagType { get { return myTagType; } }
 
-      public void publish(Stream data)
-      {
+      public void publish(Stream data) {
          long pos = data.Position;
-         if (myHandler != null)
-         {
+         if (myHandler != null) {
             myHandler.update(myTag, new Stream[] { data });
-         }
-         else
-         {
+         } else {
             myData = new byte[data.Length];
             data.Read(myData, 0, myData.Length);
          }
          data.Position = pos;
       }
 
-      public byte[] getContent()
-      {
+      public byte[] getContent() {
          return myHandler == null ? myData : myHandler.getContent(myTag);
       }
    }
@@ -54,46 +45,37 @@ namespace BCG
    /// <summary>
    /// Keeps the state of all the tags published.
    /// </summary>
-   public class AllPublishedTagState
-   {
+   public class AllPublishedTagState {
       private Dictionary<string, SingleTagHandler> myTagHandler = new Dictionary<string, SingleTagHandler>();
 
       public AllPublishedTagState() { }
 
-      public void publish(string tag, string tagType, Stream data)
-      {
+      public void publish(string tag, string tagType, Stream data) {
          if (!myTagHandler.ContainsKey(tag))
             myTagHandler.Add(tag, new SingleTagHandler(tag, tagType));
 
          myTagHandler[tag].publish(data);
       }
 
-      public void publish(string tag, string tagType, byte[] data)
-      {
+      public void publish(string tag, string tagType, byte[] data) {
          publish(tag, tagType, new MemoryStream(data));
       }
 
-      public string[] publishedTags
-      { get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } }
+      public string[] publishedTags { get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } }
 
-      public byte[] getContent(string tag, out string tagType)
-      {
+      public byte[] getContent(string tag, out string tagType) {
          tagType = myTagHandler[tag].tagType;
          return myTagHandler[tag].getContent();
       }
 
-      public bool contains(string tag)
-      {
-          return myTagHandler.ContainsKey(tag);
+      public bool contains(string tag) {
+         return myTagHandler.ContainsKey(tag);
       }
-      public void clear(string tag)
-      {
-         if (myTagHandler.ContainsKey(tag))
-         {
+      public void clear(string tag) {
+         if (myTagHandler.ContainsKey(tag)) {
             myTagHandler.Remove(tag);
-         }
-         else{}
-            // BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning);
+         } else { }
+         // BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning);
 
       }
    }

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/PubSubServerUnmanagedTCPConnection.cs

@@ -1,117 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.Runtime.InteropServices;
-using mbts;
-//using BCG.Utilities;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.Runtime.InteropServices;
+// using mbts;
+// //using BCG.Utilities;
 
-namespace BCG
-{
-   public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection
-   {
-      private BlutoSubAddress myServerAddress;
-      private SmartConnectionManagedWrapper myWrapper;
-      private ConnectionStatusChange myConnCallback;
-      private SyncQueue<Pair<int, byte[]>> myAllMsgsQueue = new SyncQueue<Pair<int, byte[]>>();
+// namespace BCG
+// {
+//    public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection
+//    {
+//       private BlutoSubAddress myServerAddress;
+//       private SmartConnectionManagedWrapper myWrapper;
+//       private ConnectionStatusChange myConnCallback;
+//       private SyncQueue<Pair<int, byte[]>> myAllMsgsQueue = new SyncQueue<Pair<int, byte[]>>();
 
-      public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper)
-      {
-         myServerAddress = new BlutoSubAddress(serverURL);
-         myWrapper = wrapper;
-         myWrapper.initConnection(new BlutoSubAddress(serverURL));
-         myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages));
-      }
+//       public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper)
+//       {
+//          myServerAddress = new BlutoSubAddress(serverURL);
+//          myWrapper = wrapper;
+//          myWrapper.initConnection(new BlutoSubAddress(serverURL));
+//          myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages));
+//       }
 
-      #region IPubSubServerConnection Members
+//       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string confId)
-      {
-         //BlueEventLog.logAssert(false, "This function should not be used on this class");
-      }
+//       public void sendWithConfirmation(byte[] data, ref string confId)
+//       {
+//          //BlueEventLog.logAssert(false, "This function should not be used on this class");
+//       }
 
-      public void send(byte[] data)
-      {
-         myWrapper.send(myServerAddress, data);
-      }
+//       public void send(byte[] data)
+//       {
+//          myWrapper.send(myServerAddress, data);
+//       }
 
-      public void send(System.IO.Stream str)
-      {
-         send (StreamUtilityMethods.getBytesFromStream(str));
-      }
+//       public void send(System.IO.Stream str)
+//       {
+//          send (StreamUtilityMethods.getBytesFromStream(str));
+//       }
 
-      public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
-      {
-         List<Pair<int, byte[]>> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements);
-         List<byte[]> rv = new List<byte[]>();
-         bool connSet = false;
-         foreach (Pair<int, byte[]> p in msgs)
-         {
-            int msgType = p.first;
-            byte[] msg = p.second;
-            if (msgType == 1)
-            {
-               rv.Add(msg);
-            }
-            else
-            {
-               connSet = true;
-               string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established");
-               Console.WriteLine(connMsg);
-               myConnCallback(myServerAddress.ToString(), msgType == 3);
-            }
-         }
-         return connSet ? new List<byte[]>() : rv;
-      }
-         
-      public string serverURL
-      {
-         get { return myServerAddress.ToString(); }
-      }
+//       public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
+//       {
+//          List<Pair<int, byte[]>> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements);
+//          List<byte[]> rv = new List<byte[]>();
+//          bool connSet = false;
+//          foreach (Pair<int, byte[]> p in msgs)
+//          {
+//             int msgType = p.first;
+//             byte[] msg = p.second;
+//             if (msgType == 1)
+//             {
+//                rv.Add(msg);
+//             }
+//             else
+//             {
+//                connSet = true;
+//                string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established");
+//                Console.WriteLine(connMsg);
+//                myConnCallback(myServerAddress.ToString(), msgType == 3);
+//             }
+//          }
+//          return connSet ? new List<byte[]>() : rv;
+//       }
 
-      public string clientURL
-      {
-         get { return myWrapper.getClientAsSeenByServer (serverURL); }
-      }
+//       public string serverURL
+//       {
+//          get { return myServerAddress.ToString(); }
+//       }
 
-      public void registerCallback(ConnectionStatusChange del)
-      {
-         myConnCallback = del;
-      }
+//       public string clientURL
+//       {
+//          get { return myWrapper.getClientAsSeenByServer (serverURL); }
+//       }
 
-      public void registerDataCallback(DataDelegate del)
-      {
-      }
+//       public void registerCallback(ConnectionStatusChange del)
+//       {
+//          myConnCallback = del;
+//       }
 
-      public void registerSendConfirmationCallback(SendConfirmationDelegate del)
-      {
-      }
+//       public void registerDataCallback(DataDelegate del)
+//       {
+//       }
 
-      #endregion
+//       public void registerSendConfirmationCallback(SendConfirmationDelegate del)
+//       {
+//       }
 
-      #region IDisposable Members
+//       #endregion
 
-      public void Dispose()
-      {
-         
-      }
+//       #region IDisposable Members
 
-      #endregion
+//       public void Dispose()
+//       {
 
-      #region IWorker Members
-      public void doWork()
-      {
-      }
-      #endregion
+//       }
 
-      /// <summary>
-      /// this is the only method that it executed on
-      /// a different thread
-      /// </summary>
-      /// <param name="data"></param>
-      /// <returns></returns>
-      private void myHandleServerMessages(int msgType, byte[] msg)
-      {
-         myAllMsgsQueue.enqueue(new Pair<int, byte[]>(msgType, msg));
-      }
-   }
-}
+//       #endregion
+
+//       #region IWorker Members
+//       public void doWork()
+//       {
+//       }
+//       #endregion
+
+//       /// <summary>
+//       /// this is the only method that it executed on
+//       /// a different thread
+//       /// </summary>
+//       /// <param name="data"></param>
+//       /// <returns></returns>
+//       private void myHandleServerMessages(int msgType, byte[] msg)
+//       {
+//          myAllMsgsQueue.enqueue(new Pair<int, byte[]>(msgType, msg));
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs

@@ -4,8 +4,7 @@ using System.Text;
 using System.IO;
 using BCG.Utilities.General;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// This is the interface that both the PublisherCore and the
    /// SubscriberCore use to communicate with the PubsubServer.
@@ -20,22 +19,23 @@ namespace BCG
    /// multiple threads need to communicate to the PubSub, they 
    /// should create their own Subscribers.
    /// </summary>
-   public interface IPubSubServerConnection : IDisposable, IWorker
-   {
+   public interface IPubSubServerConnection : IDisposable, IWorker {
       /// <summary>
       /// The delegate is fired signalling if the data send was
       /// successful or not
       /// </summary>
-      void sendWithConfirmation(byte[] data, ref string confId);
+      //void sendWithConfirmation(byte[] data, ref string confId);
       /// <summary>
       /// send is a non waiting call with no guarantee
       /// of data being delivered
       /// </summary>
-      void send (byte[] data);
-      /// <summary>
-      /// similiar to above
-      /// </summary>
+      // void send(byte[] data);
+      // /// <summary>
+      // /// similiar to above
+      // /// </summary>
       void send(Stream str);
+      void send(PubSubMessage msg);
+      void send_with_confirmation(PubSubMessage msg, ref string confId);
       /// <summary>
       /// Returns with 0 or more elements, iff the timeout
       /// expires or maxElements have been reached (selective polling),
@@ -51,13 +51,13 @@ namespace BCG
       /// <summary>
       /// server URL to which the connection is made
       /// </summary>
-      string serverURL { get;}
+      string serverURL { get; }
       /// <summary>
       /// client URL as seen by the server. if a connection has not
       /// been established (as can happen in TCP) an empty string 
       /// is returned
       /// </summary>
-      string clientURL { get;}
+      string clientURL { get; }
 
       /// <summary>
       /// registers the callback to receive 

src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs

@@ -25,48 +25,64 @@ namespace BCG
                                                    ITCPMessageHandler,
                                                    ITCPConnectionHandler
    {
-      private ITCPConnection myConnection, myTempNewConnection;
-      private BlutoSubAddress myServerAddress;
-      private ThreadSafeQueue<Pair<string, bool>> myConfs = new ThreadSafeQueue<Pair<string, bool>>();
-      private SyncQueue<byte[]> myDataQueue = new SyncQueue<byte[]>();
-      private DateTime myLastHB = DateTime.Now;
-      private ConnectionStatusChange myConnCallback;
-      private DataDelegate myDataCallback;
-      private SendConfirmationDelegate myOnSendConfirmCallback;
-      private TimeSpan myConnectionDropThreshold =
+      private ITCPConnection _connection, _new_connection;
+      private readonly BlutoSubAddress _server_address;
+      // if a message(with confirm) can not be sent , its id is queued here, 
+      // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
+      //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
+      private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
+      // in onData, data is queued here
+      private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
+      private DateTime _last_hb = DateTime.Now;
+      private ConnectionStatusChange _conn_callback;
+      private DataDelegate _data_callback;
+      private SendConfirmationDelegate _on_send_onfirm_callback;
+      private readonly TimeSpan _connection_drop_threshold =
          new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
 
       public PubSubServerManagedTCPConnection(string serverURL)
          : this(new BlutoSubAddress(serverURL)) { }
 
-      public PubSubServerManagedTCPConnection(BlutoSubAddress serverAddress)
+      public PubSubServerManagedTCPConnection(BlutoSubAddress server_address)
       {
-         myServerAddress = serverAddress;
-         ClientToServerManagedTCPConnector.connectTo(serverAddress, this);
+         _server_address = server_address;
+         _connect_to_ps_server();
+      }
+
+      private void _connect_to_ps_server()
+      {
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
+         ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
       }
 
       #region IPubSubServerConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string confId)
+      public void sendWithConfirmation(byte[] data, ref string conf_id)
       {
-         if (myConnection != null)
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
+         if (_connection != null)
          {
-            myConnection.sendWithConfirmation(data, ref confId, new SendConfirmationDelegate(myOnSendConfirm));
+            _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) =>
+            {
+               if (_on_send_onfirm_callback != null)
+                  _on_send_onfirm_callback(msg_id, confirmed);
+            });
          }
          else
          {
-            confId = Guid.NewGuid().ToString();
-            myConfs.enqueue(new Pair<string, bool>(confId, false));
+            bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
+            conf_id = Guid.NewGuid().ToString();
+            _negative_unsent_confs.enqueue(conf_id);
          }
       }
 
       public void send(byte[] data)
       {
-         Console.WriteLine($"PubSubServerManagedTCPConnection.send server: {myServerAddress} data: {data.Length} myConnection : {myConnection != null}");
-         if (myConnection != null)
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
+         if (_connection != null)
          {
-            Console.WriteLine("PubSubServerManagedTCPConnection.send server: " + myServerAddress + "data: " + data.Length);
-            myConnection.send(data);
+            bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
+            _connection.send(data);
          }
       }
 
@@ -76,14 +92,25 @@ namespace BCG
          send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
       }
 
+      public void send(PubSubMessage msg)
+      {
+         send(msg.to_stream());
+      }
+
+      public void send_with_confirmation(PubSubMessage msg, ref string conf_id)
+      {
+         var bytes = msg.to_bytes();
+         sendWithConfirmation(bytes, ref conf_id);
+      }
+
       public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
       {
-         return myDataQueue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
+         return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
       }
 
       public string serverURL
       {
-         get { return myServerAddress.ToString(); }
+         get { return _server_address.ToString(); }
       }
 
       public string clientURL
@@ -93,7 +120,7 @@ namespace BCG
             string rv = "";
             lock (this)
             {
-               if (myConnection == null)
+               if (_connection == null)
                {
                   rv = "";
                }
@@ -103,7 +130,7 @@ namespace BCG
                   {
                      string ip;
                      int port;
-                     myConnection.getSelfAddress(out ip, out port);
+                     _connection.getSelfAddress(out ip, out port);
                      rv = new BlutoSubAddress(ip, port).ToString();
                   }
                   else
@@ -118,40 +145,40 @@ namespace BCG
 
       public void registerCallback(ConnectionStatusChange del)
       {
-         myConnCallback = del;
+         _conn_callback = del;
       }
 
       public void registerDataCallback(DataDelegate del)
       {
-         myDataCallback = del;
+         _data_callback = del;
       }
 
       public void registerSendConfirmationCallback(SendConfirmationDelegate del)
       {
-         myOnSendConfirmCallback = del;
+         _on_send_onfirm_callback = del;
       }
 
       #endregion
 
       #region IDisposable Members
 
-      private bool myIsDisposed = false;
+      private bool _is_disposed = false;
       public void Dispose()
       {
-         Console.WriteLine("PubSubServerManagedTCPConnection.Dispose");
+         bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
          //throw new Exception("The method or operation is not implemented.");
-         if (myConnection != null)
+         if (_connection != null)
          {
-            (myConnection as AsyncTCPConnection).Dispose();
+            _connection.Dispose();
 
          }
-         myIsDisposed = true;
+         _is_disposed = true;
       }
       public bool isDisposed
       {
          get
          {
-            return myIsDisposed;
+            return _is_disposed;
          }
 
       }
@@ -168,17 +195,40 @@ namespace BCG
       /// <param name="message"></param>
       public void onData(RemoteMessage message)
       {
-         Console.WriteLine($"PubSubServerManagedTCPConnection.onData msg:type: {message.messageType}");
-         if (message.messageType == RemoteMessageType.data)
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
+         // TODO check for disconenction
+         switch (message.messageType)
          {
-            if (myDataCallback != null)
-               myDataCallback(message.remoteConnectionInfo.ToString(), message.message);
+            case RemoteMessageType.Data:
+               if (_data_callback != null)
+                  _data_callback(message.remoteConnectionInfo.ToString(), message.message);
+               else
+                  _in_data_queue.enqueue(message.message);
+               break;
+            case RemoteMessageType.Heartbeat:
+               _last_hb = DateTime.Now;
+               break;
+            case RemoteMessageType.Connection:
+               _conn_callback(_server_address.ToString(), true);
+               break;
+            case RemoteMessageType.Disconnection:
+               if (_conn_callback != null)
+                  _conn_callback(_server_address.ToString(), false);
+               _connection.Dispose();
+               _connect_to_ps_server();
+               break;
+
+         }
+         if (message.messageType == RemoteMessageType.Data)
+         {
+            if (_data_callback != null)
+               _data_callback(message.remoteConnectionInfo.ToString(), message.message);
             else
-               myDataQueue.enqueue(message.message);
+               _in_data_queue.enqueue(message.message);
          }
          else
          {
-            myLastHB = DateTime.Now;
+            _last_hb = DateTime.Now;
          }
       }
 
@@ -197,16 +247,56 @@ namespace BCG
          int port;
          connection.getPeerAddress(out ip, out port);
          BlutoSubAddress add = new BlutoSubAddress(ip, port);
-         Console.WriteLine($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()}");
-         if (add.ToString() == myServerAddress.ToString())
+         bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
+         if (add.ToString() == _server_address.ToString())
          {
-            myLastHB = DateTime.Now;
-            myTempNewConnection = connection;
+            _last_hb = DateTime.Now;
+            _new_connection = connection;
          }
       }
 
       #endregion
 
+      void _dispose_and_reconnect_if_missing_hb()
+      {
+         if (!_connection.isDisposed)
+         {
+            TimeSpan hb_interval = DateTime.Now - _last_hb;
+            bool hb_missed = hb_interval > _connection_drop_threshold;
+            if (hb_missed)
+            {
+               if (_conn_callback != null)
+                  _conn_callback(_server_address.ToString(), false);
+               bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
+               _connection.Dispose();
+               _connect_to_ps_server();
+            }
+         }
+         else
+         {
+            // already disposed, nothing to do
+         }
+      }
+
+      void _check_for_new_connection()
+      {
+         if (_new_connection != null)
+         {
+            _connection = _new_connection;
+            _new_connection = null; // since we are using its presence as a marker of nw connections
+            bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
+            _connection.register(this);
+            if (_conn_callback != null)
+               _conn_callback(_server_address.ToString(), true);
+            if (_on_send_onfirm_callback != null)
+            {
+               var msg_ids = _negative_unsent_confs.dequeueAll();
+               foreach (var id in msg_ids)
+                  _on_send_onfirm_callback(id, false);
+            }
+         }
+      }
+
       #region IWorker Members
 
       /// <summary>
@@ -216,55 +306,56 @@ namespace BCG
       /// </summary>
       public void doWork()
       {
+         //_dispose_and_reconnect_if_missing_hb();
+         _check_for_new_connection();
          ///check for heartbeats and let clients know in 
          ///case of missing hb (connection breaking)
-         TimeSpan curr_hb_interval = DateTime.Now - myLastHB;
-         bool hb_missed = curr_hb_interval > myConnectionDropThreshold;
-         bool conn_disposed = myConnection != null && myConnection.isDisposed;
-         if (conn_disposed || hb_missed)
-         {
-            if (hb_missed)
-            {
-               // since conn breakage was diagnosed logically, lets physically close the connection
-               myConnection.Dispose();
-               string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
-               Console.WriteLine(dispose_msg);
-            }
-            myConnection = null; // this is used as a marker for a dead connection
-            if (myConnCallback != null)
-               myConnCallback(myServerAddress.ToString(), false);
-            string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
-            $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {curr_hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
-            Console.WriteLine(reconn_msg);
-            ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
-         }
-         if (myTempNewConnection != null)
-         {
-            if (myConnection != null)
-            {
-               // TODOds
-               // log error and close it
-               // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
-               //                                                 + " before a new connection is established");
-            }
-            Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
-            myConnection = myTempNewConnection;
-            myTempNewConnection = null;
-            myConnection.register(this);
-            myConfs.dequeueAll();
-            if (myConnCallback != null)
-               myConnCallback(myServerAddress.ToString(), true);
-         }
-         foreach (Pair<string, bool> p in myConfs.dequeueAll())
-            if (myOnSendConfirmCallback != null)
-               myOnSendConfirmCallback(p.first, p.second);
+         // TimeSpan hb_interval = DateTime.Now - myLastHB;
+         // bool hb_missed = hb_interval > myConnectionDropThreshold;
+         // bool conn_disposed = myConnection != null || myConnection.isDisposed;
+         // if (conn_disposed || hb_missed)
+         // {
+         //    if (hb_missed)
+         //    {
+         //       // since conn breakage was diagnosed logically, lets physically close the connection
+         //       myConnection.Dispose();
+         //       string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
+         //       Console.WriteLine(dispose_msg);
+         //    }
+         //    myConnection = null; // this is used as a marker for a dead connection
+         //    if (myConnCallback != null)
+         //       myConnCallback(myServerAddress.ToString(), false);
+         //    string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
+         //    $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
+         //    ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
+         // }
+         // if (myTempNewConnection != null)
+         // {
+         //    if (myConnection != null)
+         //    {
+         //       // TODOds
+         //       // log error and close it
+         //       // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
+         //       //                                                 + " before a new connection is established");
+         //    }
+         //    Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
+         //    myConnection = myTempNewConnection;
+         //    myTempNewConnection = null;
+         //    myConnection.register(this);
+         //    myConfs.dequeueAll();
+         //    if (myConnCallback != null)
+         //       myConnCallback(myServerAddress.ToString(), true);
+         // }
+         // foreach (Pair<string, bool> p in myConfs.dequeueAll())
+         //    if (myOnSendConfirmCallback != null)
+         //       myOnSendConfirmCallback(p.first, p.second);
       }
 
       #endregion
 
-      private void myOnSendConfirm(string messageId, bool confirmed)
-      {
-         myConfs.enqueue(new Pair<string, bool>(messageId, confirmed));
-      }
+      // private void myOnSendConfirm(string messageId, bool confirmed) {
+      //    _negative_confs.enqueue(new Pair<string, bool>(messageId, confirmed));
+      // }
+
    }
 }

src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs

@@ -7,144 +7,116 @@ using System.Text;
 
 
 using BCG;
-//using BCG.Utilities;
+using System.Collections.Generic;
 
-namespace BCG
-{
-   public class MetaData : ICloneable
-   {
-      private Hashtable myHash = new Hashtable();
+namespace BCG {
+   // Thhis "wraps" around the following 4 PubSub meta objects, serializes it on the client and 
+   // deserialize it on the pubsub 
+   // PubMetaData
+   // SubMetaData
+   // TransactionMetaData
+   // UnSubMetaData
+   public class MetaData : ICloneable {
+      private IDictionary _name_value_map = new Dictionary<string, string>();
       int mySize = -1;
 
-      public MetaData()
-      {}
+      public MetaData() { }
 
-      public MetaData(object o)
-      {copyFrom(o);}
+      public MetaData(object o) { copyFrom(o); }
 
-      public MetaData(Stream s)
-      {
+      public MetaData(Stream s) {
          long initPos = s.Position;
          load(s);
          mySize = Convert.ToInt32(s.Position - initPos);
       }
 
-      public string this[string name]
-      {
-         get 
-         {return (string) myHash[name];}
-         set 
-         {
+      public string this[string name] {
+         get { return (string)_name_value_map[name]; }
+         set {
             if (value != null)
-               myHash[name] = value;
+               _name_value_map[name] = value;
          }
       }
 
-      public void add(string name, string [] values)
-      {
+      private void _add_string_array(string name, string[] values) {
          StringBuilder sb = new StringBuilder();
-         foreach (string s in values)
-         {
+         foreach (string s in values) {
             if (sb.Length != 0)
                sb.Append(" ");
             sb.Append(s);
          }
-         myHash[name] = sb.ToString();
+         _name_value_map[name] = sb.ToString();
       }
 
-      public int getAsInt(string name)
-      {return Int32.Parse(this[name]);}
+      private int getAsInt(string name) { return Int32.Parse(this[name]); }
 
-      public string[] getAsStringArray (string name)
-      {
+      private string[] _get_as_string_array(string name) {
          if (this[name] != "" && this[name] != null)
             return this[name].Split(' ');
          else
             return new string[0];
       }
 
-      public int size()
-      {
-         if (mySize < 0)
-         {
+      public int size() {
+         if (mySize < 0) {
             MemoryStream ms = new MemoryStream();
             this.save(ms);
             return mySize = (int)ms.Position;
-         }
-         else
+         } else
             return mySize;
       }
 
-      public bool isEqual (MetaData md)
-      {
-         if (md.myHash.Keys.Count != myHash.Keys.Count)
+      private bool isEqual(MetaData md) {
+         if (md._name_value_map.Keys.Count != _name_value_map.Keys.Count)
             return false;
 
-         foreach (string k in myHash.Keys)
-            if (!myHash[k].Equals(md.myHash[k]))
+         foreach (string k in _name_value_map.Keys)
+            if (!_name_value_map[k].Equals(md._name_value_map[k]))
                return false;
          return true;
       }
 
-      public void load(Stream s)
-      {
-         myHash = new Hashtable();
-         string [] state = 
+      // loads the content of the stream into this instance's map
+      private void load(Stream s) {
+         _name_value_map = new Hashtable();
+         string[] state =
             new BinaryReader(s).ReadString().Split('\n');
 
-         for (int i = 0; i < state.Length-1; i+= 2)
-            myHash[state[i]] = state[i+1];
+         for (int i = 0; i < state.Length - 1; i += 2)
+            _name_value_map[state[i]] = state[i + 1];
       }
 
-      public void save(Stream s)
-      {
+      // writes the content of this instance to stream s
+      public void save(Stream s) {
          new BinaryWriter(s).Write(this.ToString());
       }
 
-      public override string ToString()
-      {
+      public override string ToString() {
          StringBuilder sb = new StringBuilder();
-         foreach (string key in myHash.Keys) 
-            sb.AppendFormat("{0}\n{1}\n",key,myHash[key]);
+         foreach (string key in _name_value_map.Keys)
+            sb.AppendFormat("{0}\n{1}\n", key, _name_value_map[key]);
          return sb.ToString();
       }
 
-      public void copyFrom(object o)
-      {
+      // iterates through the fields and enters the name, values into _hash
+      public void copyFrom(object o) {
          this["op-code"] = o.GetType().Name;
-
-         
-
-         foreach (FieldInfo fi in o.GetType().GetFields())
-         {
-         
+         foreach (FieldInfo fi in o.GetType().GetFields()) {
             if (fi.GetValue(o) == null)
                continue;
-            
             else if (fi.GetValue(o) is String[])
-              add(fi.Name, (string[])fi.GetValue(o));
-            
+               _add_string_array(fi.Name, (string[])fi.GetValue(o));
             else
-               myHash[fi.Name] = fi.GetValue(o).ToString();
-            
+               _name_value_map[fi.Name] = fi.GetValue(o).ToString();
+
          }
       }
-
-      public void copyTo(object o)
-      {
-         
-         
-         foreach (FieldInfo fi in o.GetType().GetFields()) 
-         {
-         
+      // copies the name value map into object o
+      public void copyTo(object o) {
+         foreach (FieldInfo fi in o.GetType().GetFields()) {
             string value = this[fi.Name];
-            //this should only be here temporarily, while the new decompression changes are waiting to go into prod. 
-            if (value == null)
-            {
-               //BlueEventLog.logEvent("Error: Metadata missing field " + fi.Name, BlueEventLog2Level.MissingMetadata);
-            }
-            else
-            {
+            if (value == null) {
+            } else {
                if (fi.FieldType.Equals(typeof(string)))
                   fi.SetValue(o, value);
                else if (fi.FieldType.Equals(typeof(int)))
@@ -167,11 +139,10 @@ namespace BCG
          }
       }
 
-      public object Clone()
-      {
+      public object Clone() {
          MemoryStream ms = new MemoryStream();
          this.save(ms);
-         ms.Seek(0,SeekOrigin.Begin);
+         ms.Seek(0, SeekOrigin.Begin);
          return new MetaData(ms);
       }
    }

src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs

@@ -1,25 +1,25 @@
-using System;
+// using System;
 
-namespace BCG
-{
-   [Serializable]
-   public class RenewMetaData 
-   {
-      public string subscriber;
-      public int lease;
-      public bool c;
-      
-      public SubAddress getSubAddress()
-      {return SubAddress.decode(subscriber);}
+// namespace BCG
+// {
+//    [Serializable]
+//    public class RenewMetaData 
+//    {
+//       public string subscriber;
+//       public int lease;
+//       public bool c;
 
-      public RenewMetaData()
-      {}
+//       public SubAddress getSubAddress()
+//       {return SubAddress.decode(subscriber);}
 
-      public RenewMetaData(string s, 
-         int lease)
-      {
-         this.subscriber = s;
-         this.lease = lease;
-      }
-   }
-}
+//       public RenewMetaData()
+//       {}
+
+//       public RenewMetaData(string s, 
+//          int lease)
+//       {
+//          this.subscriber = s;
+//          this.lease = lease;
+//       }
+//    }
+// }

src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs

@@ -3,37 +3,33 @@ using System.Collections;
 using System.Reflection;
 using BCG.Utilities;
 using System.Collections.Generic;
+using System.Reactive.Subjects;
 
-namespace BCG
-{
-	public class TagType2HandlerMap
-	{
+namespace BCG {
+   public class TagType2HandlerMap {
       private static TagType2HandlerMap myServerHandlerMap = null,
                                         myPublisherStateMap = null;
-      private Dictionary<string, IPubSubServerTagHandler> myTagTypeToPubHandlerMap = new Dictionary<string, IPubSubServerTagHandler>();
+      private Dictionary<string, IPubSubServerTagHandler> myTagTypeToPubHandlerMap =
+         new Dictionary<string, IPubSubServerTagHandler>();
 
-      public TagType2HandlerMap()
-      {
-         if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null)
-         {
-            foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries)
-            {
-               try
-               {
+      public TagType2HandlerMap() {
+         if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null) {
+            foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries) {
+               try {
+                  var ass_loc = assembly_location;
+                  bts.Log.Gen.info($"TagType2HandlerMap entry: {entry} assembly_location: {ass_loc}");
                   Assembly assem = Assembly.LoadFrom(entry.assembly);
-                  Type t = assem.GetType(entry.handlerClass, true, false);
+                  Type t = assem.GetType(entry.handlerClass, true, true);
                   object obj = Activator.CreateInstance(t);
                   IPubSubServerTagHandler tagHandler = obj as IPubSubServerTagHandler;
+                  bts.Log.Gen.info($"TagType2HandlerMap successfully created obj: {obj} of type: {t} from assem: {assem}");
                   string[] tagTypes = tagHandler.supportedTagTypes;
-                  foreach (string tagType in tagTypes)
-                  {
+                  foreach (string tagType in tagTypes) {
+                     bts.Log.Gen.info($"TagType2HandlerMap adding tagType: {tagType} tagHandler: {tagHandler}");
                      myTagTypeToPubHandlerMap[tagType] = tagHandler;
                   }
-               }
-               catch (Exception ex)
-               {
-                  // BlueEventLog.logEvent(string.Format("error happened while loading type from the assembly ", entry.handlerClass, entry.assembly), BlueEventLevel.Warning);
-                  // BlueEventLog.logEvent(ex);
+               } catch (Exception ex) {
+                  bts.Log.Gen.error($"TagType2HandlerMap threw exception ex: {ex}");
                   if (ex is OutOfMemoryException)
                      throw ex;
                }
@@ -41,30 +37,41 @@ namespace BCG
          }
       }
 
-      private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType)
-      {
+      public static string assembly_location {
+         get {
+            // string codeBase = Assembly.GetExecutingAssembly().CodeBase;
+            // UriBuilder uri = new UriBuilder(codeBase);
+            // string path = Uri.UnescapeDataString(uri.Path);
+            // return Path.GetDirectoryName(path);
+            // Get the executing assembly
+            Assembly executingAssembly = Assembly.GetExecutingAssembly();
+
+            // Get the location (path) of the executing assembly
+            string assembly_location = executingAssembly.Location;
+
+            return assembly_location;
+         }
+      }
+
+      private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType) {
          IPubSubServerTagHandler rv;
          myTagTypeToPubHandlerMap.TryGetValue(tagType, out rv);
+         bts.Log.Gen.info($"TagType2HandlerMap.myGetHandlerFromTagType tag_type: {tagType} rv: {rv}");
          return rv;
       }
 
-      public static IPubSubServerTagHandler getHandler(string type, bool isServer)
-      {
-         lock (typeof(IPubSubServerTagHandler))
-         {
-            if (isServer)
-            {
+      public static IPubSubServerTagHandler getHandler(string type, bool isServer) {
+         lock (typeof(IPubSubServerTagHandler)) {
+            if (isServer) {
                if (myServerHandlerMap == null)
                   myServerHandlerMap = new TagType2HandlerMap();
                return myServerHandlerMap.myGetHandlerFromTagType(type);
-            }
-            else
-            {
+            } else {
                if (myPublisherStateMap == null)
                   myPublisherStateMap = new TagType2HandlerMap();
                return myPublisherStateMap.myGetHandlerFromTagType(type);
             }
          }
       }
-	}
+   }
 }

src/DataStreamSuite/PubSub/Client/TagTypeToAssembly.cs

@@ -1,17 +1,18 @@
 using System;
 
-namespace BCG
-{
+namespace BCG {
    [Serializable]
-   public class TagTypeToAssembly
-   {
+   public class TagTypeToAssembly {
       //[Serializable]
-      public class Entry 
-      {
-         public string handlerClass;
-         public string assembly;
+      public class Entry {
+         public string handlerClass = "BCG.PubSubServerTagHandler";
+         // TODOds
+         public string assembly = "C:\\Users\\m_tah\\spark\\build\\Debug\\bin\\net7.0\\DataStreamCore.dll";
+         public override string ToString() {
+            return $"handlerClass: {handlerClass} assembly: {assembly}";
+         }
       }
 
-      public Entry[] entries;
+      public Entry[] entries = new Entry[] { new Entry() };
    }
 }

src/DataStreamSuite/PubSub/Config/PubSub3Config.cs

@@ -5,17 +5,13 @@ using System.Text.RegularExpressions;
 //using BCG.Utilities.BlueConfiguration;
 //using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    public class PubSub3Config //: ConfigurationRequest
    {
       private static PubSub3Config myInstance;
-      public static PubSub3Config instance
-      {
-         get
-         {
-            if (myInstance == null)
-            {
+      public static PubSub3Config instance {
+         get {
+            if (myInstance == null) {
                myInstance = new PubSub3Config();
                //myInstance = (PubSub3Config) ConfigurationRequester.request(typeof(PubSub3Config));
             }
@@ -43,7 +39,7 @@ namespace BCG
       private string[] myPS3_CrushableTags = new string[] { };
 
       //moved from PS2
-      private TagTypeToAssembly myTagTypeToAssemblyMap;
+      private TagTypeToAssembly myTagTypeToAssemblyMap = new TagTypeToAssembly();
       private string myPS3_ServerQueue;
 
       //substation stuff
@@ -83,122 +79,101 @@ namespace BCG
 
       // bluecomm transition
       //public ConfigKeys.BlueComm_PSTransition BlueComm_PSTransition { get; set; }
-      public PubSub3Config()
-      {
+      public PubSub3Config() {
          //BlueComm_PSTransition = new BCG.ConfigKeys.BlueComm_PSTransition(true);
       }
 
-      public bool isValid()
-      {
+      public bool isValid() {
          return true;
       }
 
-      public int PS3_retriesOnNoBufferSpace
-      {
+      public int PS3_retriesOnNoBufferSpace {
          get { return myPS3_retriesOnNoBufferSpace; }
          set { myPS3_retriesOnNoBufferSpace = value; }
       }
 
-      public int PS3_millisecondsBetweenNoBufferSpaceRetries
-      {
+      public int PS3_millisecondsBetweenNoBufferSpaceRetries {
          get { return myPS3_millisecondsBetweenNoBufferSpaceRetries; }
          set { myPS3_millisecondsBetweenNoBufferSpaceRetries = value; }
       }
 
-      public int PS3_millisecondsBetweenTimeSliceUpdates
-      {
+      public int PS3_millisecondsBetweenTimeSliceUpdates {
          get { return myPS3_millisecondsBetweenTimeSliceUpdates; }
          set { myPS3_millisecondsBetweenTimeSliceUpdates = value; }
       }
 
 
-      public bool BlueComm_mustDie
-      {
+      public bool BlueComm_mustDie {
          get { return myBlueComm_mustDie; }
          set { myBlueComm_mustDie = value; }
       }
 
-      public int PS3_HeartBeatInterval
-      {
+      public int PS3_HeartBeatInterval {
          get { return myPS3_HeartBeatInterval; }
          set { myPS3_HeartBeatInterval = value; }
       }
 
-      public int PS3_DoItAllSecondsBetweenLogs
-      {
+      public int PS3_DoItAllSecondsBetweenLogs {
          get { return myPS3_DoItAllSecondsBetweenLogs; }
          set { myPS3_DoItAllSecondsBetweenLogs = value; }
       }
 
-      public int PS3_MissingHeartBeatsForGC
-      {
+      public int PS3_MissingHeartBeatsForGC {
          get { return myPS3_MissingHeartBeatsBeforeGC; }
          set { myPS3_MissingHeartBeatsBeforeGC = value; }
       }
 
-      public bool PS3_UseManagedSockets
-      {
+      public bool PS3_UseManagedSockets {
          get { return myPS3_UseManagedSockets; }
          set { myPS3_UseManagedSockets = value; }
       }
 
-      public int PS3_StateUpdatePeriod
-      {
+      public int PS3_StateUpdatePeriod {
          get { return myPS3_StateUpdatePeriod; }
          set { myPS3_StateUpdatePeriod = value; }
       }
 
-      public int PS3_StatisticsUpdatePeriod
-      {
+      public int PS3_StatisticsUpdatePeriod {
          get { return myPS3_StatisticsUpdatePeriod; }
          set { myPS3_StatisticsUpdatePeriod = value; }
       }
 
-      public int PS3_DisposedConnectionCheckPeriod
-      {
+      public int PS3_DisposedConnectionCheckPeriod {
          get { return myPS3_DisposedConnectionCheckPeriod; }
          set { myPS3_DisposedConnectionCheckPeriod = value; }
       }
 
-      public int PS3_MessageProcessingWarningThreshold
-      {
+      public int PS3_MessageProcessingWarningThreshold {
          get { return myPS3_MessageProcessingWarningThreshold; }
          set { myPS3_MessageProcessingWarningThreshold = value; }
       }
 
-      public int PS3_millisecondsBetweenSubInfoUpdates
-      {
+      public int PS3_millisecondsBetweenSubInfoUpdates {
          get { return myPS3_millisecondsBetweenSubInfoUpdates; }
          set { myPS3_millisecondsBetweenSubInfoUpdates = value; }
       }
 
-      public double PS3_LatencyReportThresholdMS
-      {
+      public double PS3_LatencyReportThresholdMS {
          get { return myPS3_LatencyReportThresholdMS; }
          set { myPS3_LatencyReportThresholdMS = value; }
       }
 
-      public int PS3_ServerPort
-      {
+      public int PS3_ServerPort {
          get { return myPS3_ServerPort; }
          set { myPS3_ServerPort = value; }
       }
 
-      public PubSubRoute[] getSubRouting()
-      { return getRouting(); }
+      public PubSubRoute[] getSubRouting() { return getRouting(); }
 
-      public PubSubRoute[] getPubRouting()
-      { return getRouting(); }
+      public PubSubRoute[] getPubRouting() { return getRouting(); }
 
-      public PubSubRoute[] getRouting()
-      {
+      public PubSubRoute[] getRouting() {
          // TODO
          return new PubSubRoute[] { };
          // return getRouting(SitesCore.getMySiteName());
       }
 
-      public PubSubRoute[] getRouting(string forSite)
-      {
+      public PubSubRoute[] getRouting(string forSite) {
          // TODO
          return new PubSubRoute[] { };
          // if (sites_layout != null) {
@@ -214,14 +189,12 @@ namespace BCG
          return null;
       }
 
-      public List<string> getAllPubSubs()
-      {
+      public List<string> getAllPubSubs() {
          return new List<string>();
          //return getAllPubSubs(SitesCore.getMySiteName());
       }
 
-      public List<string> getAllPubSubs(string forSite)
-      {
+      public List<string> getAllPubSubs(string forSite) {
          List<string> result = new List<string>();
          foreach (PubSubRoute r in getRouting(forSite))
             result.AddRange(r.getAllPubSubs());
@@ -233,17 +206,14 @@ namespace BCG
 
       // public ConfigKeys.sites_siteName sites_siteName { get; set; }
 
-      public TagTypeToAssembly PS3_TagTypeToAssemblyMap
-      {
+      public TagTypeToAssembly PS3_TagTypeToAssemblyMap {
          get { return myTagTypeToAssemblyMap; }
          set { myTagTypeToAssemblyMap = value; }
       }
 
       //[Description("PUBSUB: The Message queue to look for pub sub messages")]
-      public string PS3_ServerQueue
-      {
-         get
-         {
+      public string PS3_ServerQueue {
+         get {
             if (myPS3_ServerQueue == null || myPS3_ServerQueue == "")
                return "PubSubQueue";
             else
@@ -252,14 +222,12 @@ namespace BCG
          set { myPS3_ServerQueue = value; }
       }
 
-      public bool PS3_InternallyPublishSubInfo
-      {
+      public bool PS3_InternallyPublishSubInfo {
          get { return myPS3_InternallyPublishSubInfo; }
          set { myPS3_InternallyPublishSubInfo = value; }
       }
 
-      public string[] PS3_CrushableTags
-      {
+      public string[] PS3_CrushableTags {
          get { return myPS3_CrushableTags; }
          set { myPS3_CrushableTags = value; }
       }
@@ -299,95 +267,80 @@ namespace BCG
 
       // }
 
-      public bool PS3_UseSubstations
-      {
+      public bool PS3_UseSubstations {
          get { return myPS3_UseSubstations; }
          set { myPS3_UseSubstations = value; }
       }
 
-      public PubSubPortRange PS3_PortRange
-      {
+      public PubSubPortRange PS3_PortRange {
          get { return myPS3_PortRange; }
          set { myPS3_PortRange = value; }
       }
 
-      public bool PS3_BindToRandomPort
-      {
+      public bool PS3_BindToRandomPort {
          get { return myPS3_BindToRandomPort; }
          set { myPS3_BindToRandomPort = value; }
 
       }
 
-      public bool PS3_IsSubstation
-      {
+      public bool PS3_IsSubstation {
          get { return myPS3_IsSubstation; }
          set { myPS3_IsSubstation = value; }
 
       }
 
-      public bool PS3_UseDynamicClients
-      {
+      public bool PS3_UseDynamicClients {
          get { return myPS3_UseDynamicClients; }
          set { myPS3_UseDynamicClients = value; }
       }
 
-      public bool PS3_UseManagedClientsInNativeCode
-      {
+      public bool PS3_UseManagedClientsInNativeCode {
          get { return myPS3_UseManagedClientsInNativeCode; }
          set { myPS3_UseManagedClientsInNativeCode = value; }
       }
 
-      public string PS3_CompressedTagsRegex
-      {
+      public string PS3_CompressedTagsRegex {
          get { return myPS3_CompressedTagsRegex; }
 
          set { myPS3_CompressedTagsRegex = value; }
       }
 
-      public string PS3_DebugTags
-      {
+      public string PS3_DebugTags {
          get { return myPS3_DebugTags; }
          set { myPS3_DebugTags = value; }
       }
 
-      public string PS3_TagsToReportCompressionRatio
-      {
+      public string PS3_TagsToReportCompressionRatio {
          get { return myPS3_TagsToReportCompressionRatio; }
          set { myPS3_TagsToReportCompressionRatio = value; }
       }
 
-      public bool PS3_DisableMSMQ
-      {
+      public bool PS3_DisableMSMQ {
          get { return myPS3_DisableMSMQ; }
          set { myPS3_DisableMSMQ = true; }
       }
 
-      public string PS3_IPTranslation
-      {
+      public string PS3_IPTranslation {
          get { return myPS3_IPTranslation; }
          set { myPS3_IPTranslation = value; }
       }
 
-      public long PS3_GCActivationThreshold
-      {
+      public long PS3_GCActivationThreshold {
          get { return myGCActivationThreshold; }
          set { myGCActivationThreshold = value; }
       }
 
-      public int PS3_WaitPeriodAfterGC
-      {
+      public int PS3_WaitPeriodAfterGC {
          get { return myWaitPeriodAfterGC; }
          set { myWaitPeriodAfterGC = value; }
       }
 
-      public bool PS3_UsePrepublishedServerState
-      {
+      public bool PS3_UsePrepublishedServerState {
          get { return myPS3_UsePrepublishedServerState; }
          set { myPS3_UsePrepublishedServerState = value; }
       }
 
-      public int PS3_MaxPacketSize
-      {
+      public int PS3_MaxPacketSize {
          get { return myPS3_MaxPacketSize; }
          set { myPS3_MaxPacketSize = value; }
       }
@@ -404,21 +357,17 @@ namespace BCG
       //    set { myRecovery_PubSubCopies = value; }
       // }
 
-      public bool isTagDebugTag(string tag)
-      {
-         if (myDebugTagsRegex == null)
-         {
+      public bool isTagDebugTag(string tag) {
+         if (myDebugTagsRegex == null) {
             myDebugTagsRegex = new Regex(myPS3_DebugTags, RegexOptions.IgnoreCase);
          }
          return myDebugTagsRegex.IsMatch(tag);
 
       }
 
-      public bool isTagCompressed(string tag)
-      {
+      public bool isTagCompressed(string tag) {
 
-         if (myCompressedTagsRegex == null)
-         {
+         if (myCompressedTagsRegex == null) {
             myCompressedTagsRegex = new Regex(myPS3_CompressedTagsRegex, RegexOptions.IgnoreCase);
 
          }
@@ -426,10 +375,8 @@ namespace BCG
          return (myCompressedTagsRegex.IsMatch(tag));
       }
 
-      public bool shouldReportTagsCompressionRatio(string tag)
-      {
-         if (myTagsToReportCompressionRatioRegex == null)
-         {
+      public bool shouldReportTagsCompressionRatio(string tag) {
+         if (myTagsToReportCompressionRatioRegex == null) {
             myTagsToReportCompressionRatioRegex = new Regex(myPS3_TagsToReportCompressionRatio, RegexOptions.IgnoreCase);
          }
          return (myTagsToReportCompressionRatioRegex.IsMatch(tag));

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs

@@ -45,8 +45,7 @@ namespace BCG
          public SendConfirmationDelegate onSendDel;
 
          public AsyncMessage(byte[] theBuffer, int theOffset)
-            : this(theBuffer, theOffset, String.Empty, null)
-         { }
+            : this(theBuffer, theOffset, String.Empty, null) { }
 
          public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel)
          {
@@ -445,7 +444,7 @@ namespace BCG
                               myDataReceived = true;
                               Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}");
                            }
-                           myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.data, myPeer));
+                           myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer));
 
                         }
                         else if (msgType == 1)
@@ -456,7 +455,7 @@ namespace BCG
                               Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}");
                            }
                            myLastHBReceivedTime = DateTime.Now;
-                           myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.heartbeat, myPeer));
+                           myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer));
                         }
                         myReadSize();
                      }

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs

@@ -0,0 +1,296 @@
+using BCG;
+
+using Google.Protobuf;
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+using System.Xml.Serialization;
+using System.Reflection;
+using BCG.Utilities.General;
+using System.Net.WebSockets;
+
+namespace BCG
+{
+
+   public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable
+   {
+      //private Socket _socket;
+      // tied to the socket above
+      private NetworkStream _nw_stream;
+      private readonly ConnectionSide _side;
+      ITCPMessageHandler _message_handler;
+      private int _msg_id = 0;
+      RemoteConnectionInfo _peer_address;
+      RemoteConnectionInfo _self_address;
+      private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
+      private DateTime _last_hb_sent = DateTime.MinValue;
+      private DateTime _last_hb_received = DateTime.MaxValue;
+      private static byte _msg_type_data = 1;
+      private static byte _msg_type_hb = 0;
+      private bool _is_disposed = false;
+      SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
+
+      public class QueuedMessage
+      {
+         public string id;
+         public SendConfirmationDelegate del;
+         public byte[] data;
+
+         public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data)
+         {
+            id = id;
+            del = del;
+            data = data;
+         }
+
+         public QueuedMessage(byte[] data)
+         : this(String.Empty, null, data) { }
+
+      }
+
+      public enum ConnectionSide
+      {
+         Client,
+         Server
+      }
+
+      public AsyncTCPConnection2(Socket socket, ConnectionSide side)
+      {
+         //_socket = socket;
+         _nw_stream = new NetworkStream(socket);
+         _side = side;
+         IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
+         _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
+
+         var loc_ep = socket.LocalEndPoint as IPEndPoint;
+         _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
+      }
+
+      public void register(ITCPMessageHandler message_handler)
+      {
+         _message_handler = message_handler;
+         Task.Run(async () => _keep_reading());
+         Task.Run(async () => await _keep_sending());
+         //_keep_reading();
+         // if add a _keep_sending threadis added then we can maek send a non async
+         // methid making it much easier to integrate with the rest
+      }
+
+      // this thread pulls 
+      private async Task _keep_sending()
+      {
+         bts.Log.Gen.info($"TCPConnection._keep_sending starting");
+         while (!_is_disposed)
+         {
+            try
+            {
+               QueuedMessage m = _data_send_queue.dequeue(100);
+               if (m == null)
+               {
+                  continue;
+               }
+               await send_async(m.data);
+               if (String.IsNullOrEmpty(m.id))
+               {
+                  m.del(m.id, true);
+               }
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
+               await Task.Delay(100);
+               _handle_error(ex);
+            }
+         }
+      }
+
+      private async Task _keep_reading()
+      {
+         bts.Log.Gen.info($"TCPConnection._keep_reading starting");
+         while (!_is_disposed)
+         {
+            try
+            {
+               byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
+               bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
+               await _handle_msg(msg);
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
+               await Task.Delay(100);
+               _handle_error(ex);
+            }
+         }
+      }
+
+      private async Task _handle_msg(byte[] msg)
+      {
+         // read the first byte to read the msg type and deal with it accordingly
+         BinaryReader br = new BinaryReader(new MemoryStream(msg));
+         byte msg_type = br.ReadByte();
+         if (msg_type == _msg_type_data)
+         {
+            byte[] data = br.ReadBytes(msg.Length - 1);
+            RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
+            _message_handler.onData(m);
+         }
+         else
+         {
+            RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
+            _message_handler.onData(m);
+         }
+      }
+
+      // not very efficient
+      private static byte[] _prefix_msg_type(byte[] data)
+      {
+         MemoryStream ms = new MemoryStream();
+         BinaryWriter bw = new BinaryWriter(ms);
+         bw.Write(_msg_type_data);
+         bw.Write(data);
+         ms.Position = 0;
+         return ms.ToArray();
+      }
+
+      private async void _handle_error(Exception ex)
+      {
+         bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
+         try
+         {
+            var socket = _nw_stream.Socket;
+            socket.Close();
+            socket.Dispose();
+         }
+         catch (Exception ex2)
+         {
+            bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
+         }
+         _is_disposed = true;
+         RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
+         _message_handler.onData(rm);
+      }
+
+      public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
+
+      public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del)
+      {
+         try
+         {
+            conf_id = (++_msg_id).ToString();
+            await TCPSocketUtils.send_bytes_async(_nw_stream, data);
+            del(conf_id, true);
+         }
+         catch (Exception ex)
+         {
+            _handle_error(ex);
+         }
+      }
+
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
+      {
+         conf_id = (++_msg_id).ToString();
+         send(data);
+         del(conf_id, true);
+      }
+
+      public void send(byte[] data)
+      {
+         Task.Run(async () => await send_async(data));
+         bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
+         //_data_send_queue.enqueue(new QueuedMessage(data));
+      }
+
+      public async Task send_async(byte[] data)
+      {
+         // prefix 4 bytes for type (data vs hb)
+         MemoryStream str = new MemoryStream();
+         BinaryWriter bw = new BinaryWriter(str);
+         bw.Write(_msg_type_data);
+         bw.Write(data);
+         try
+         {
+            await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
+         }
+         catch (Exception ex)
+         {
+            bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
+            _handle_error(ex);
+         }
+      }
+
+      public void getPeerAddress(out string ip, out int port)
+      {
+         ip = _peer_address.ip;
+         port = _peer_address.port;
+         ///.Net has some problems getting the port, so using
+         ///native code for that
+         //sockaddr addr = new sockaddr();
+         //int namelen = 16;
+         //getsockname(mySock.Handle, ref addr, ref namelen);
+         //port = addr.sin_port;
+      }
+
+      public RemoteConnectionInfo get_peer_address()
+      {
+         string ip; int port;
+         getPeerAddress(out ip, out port);
+         return new RemoteConnectionInfo(ip, port);
+      }
+
+      public RemoteConnectionInfo get_self_address()
+      {
+         string ip; int port;
+         getSelfAddress(out ip, out port);
+         return new RemoteConnectionInfo(ip, port);
+      }
+
+
+      public void getSelfAddress(out string ip, out int port)
+      {
+         ip = _self_address.ip;
+         port = _self_address.port;
+      }
+
+      private async void _send_hb()
+      {
+         if (_last_hb_sent - DateTime.Now > _hb_period)
+         {
+            MemoryStream ms = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(ms);
+            bw.Write(_msg_type_hb);
+            ms.Position = 0;
+            byte[] hb_msg = ms.ToArray();
+            await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
+         }
+      }
+
+      public void doWork()
+      {
+         _send_hb();
+      }
+
+      public void Dispose()
+      {
+         _nw_stream.Close();
+         _nw_stream.Dispose();
+         _is_disposed = true;
+      }
+
+      public bool isDisposed
+      {
+         get
+         {
+            return _is_disposed;
+         }
+      }
+
+   }
+}

src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs

@@ -4,21 +4,19 @@ using System.Text;
 using System.Net;
 using System.Net.Sockets;
 using System.Threading;
+using System.Threading.Tasks;
 //using BCG.Utilities;
 //using BCG.Utilities.General;
 
-namespace BCG
-{
-   internal class SocketConnectionArgs
-   {
+namespace BCG {
+   internal class SocketConnectionArgs {
       public EndPoint ep;
       public Socket sock;
       public ITCPConnectionHandler handler;
       public string serverURL;
 
       internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
-         string theServerURL)
-      {
+         string theServerURL) {
          ep = theEp;
          sock = theSock;
          handler = theHandler;
@@ -26,46 +24,46 @@ namespace BCG
       }
    }
 
-   public class ClientToServerManagedTCPConnector
-   {
+   public class ClientToServerManagedTCPConnector {
       private static Random myRnd = new Random(DateTime.Now.Millisecond);
       private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
       private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
                            myWaitTimeBeforeRetryingLowerLimit = 1000;
 
-      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler)
-      {
-         // TODOds
-         // BlueEventLog.logEvent(String.Format("Connection being attempted to server {0}", serverAddress.ToString()),
-         //                         PS3LogLevels.connection);
+      public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
          Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
          Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
          EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
          sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
       }
 
-      private static void myOnConnect(IAsyncResult res)
-      {
+      public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+      }
+
+      public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
+         bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
+         Task.Run(async () => {
+            await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
+         });
+      }
+
+      private static void myOnConnect(IAsyncResult res) {
          Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
          SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
-         try
-         {
+         try {
             args.sock.EndConnect(res);
-         }
-         catch (SocketException sex)
-         {
+         } catch (SocketException sex) {
             string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
             //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
             //Console.WriteLine(errMsg);
-            if (args.handler.isDisposed)
-            {
+            if (args.handler.isDisposed) {
                string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
                //Console.WriteLine(doneMsg);
                //               BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
                return;
-            }
-            else
-            {
+            } else {
                string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
                //Console.WriteLine(retryMsg);
                //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);

src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs

@@ -2,15 +2,13 @@ using System;
 using System.Collections.Generic;
 using System.Text;
 
-namespace BCG
-{
+namespace BCG {
    public delegate void SendConfirmationDelegate(string messageId, bool confirmed);
    /// <summary>
    /// This is the very basic connection that sits right 
-   /// on top of the sockets (unmanaged, managed whatever).
+   /// on top of the sockets (used on both server and client)
    /// </summary>
-   public interface ITCPConnection : IDisposable
-   {
+   public interface ITCPConnection : IDisposable {
       /// <summary>
       /// The delegate is fired signalling if the data send was
       /// successful or not
@@ -18,7 +16,7 @@ namespace BCG
       /// <param name="data"></param>
       /// <param name="confId"></param>
       /// <param name="del"></param>
-      void sendWithConfirmation (byte[] data, ref string confId, SendConfirmationDelegate del);
+      void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
       /// <summary>
       /// sends data to the other side. theres no guaranatee that
       /// data is delievered.
@@ -54,6 +52,6 @@ namespace BCG
       /// hence this is an indicator of whether the connection is
       /// alive or dead
       /// </summary>
-      bool isDisposed { get;}
+      bool isDisposed { get; }
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs

@@ -2,46 +2,51 @@ using System;
 using System.Collections.Generic;
 using System.Text;
 using System.Net;
+using bts.utils;
 
-namespace BCG
-{
-   public class RemoteConnectionInfo
-   {
+namespace BCG {
+   public class RemoteConnectionInfo {
       private string myIp;
       private int myPort;
       private IPEndPoint myEndPoint;
 
-      public RemoteConnectionInfo(string ip, int port)
-      {
+      public RemoteConnectionInfo(string ip, int port) {
          myIp = ip;
          myPort = port;
       }
 
       public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint)
-         : this (ip, port)
-      {
+         : this(ip, port) {
          myEndPoint = endPoint;
       }
 
-      public string ip
-      {
+      public string ip {
          get { return myIp; }
       }
 
-      public int port
-      {
+      public int port {
          get { return myPort; }
       }
 
-      public IPEndPoint endPoint
-      {
+      public IPEndPoint endPoint {
          get { return myEndPoint; }
       }
 
-      public override string ToString()
-      {
+      public override string ToString() {
          return ip + ":" + port;
       }
-      
+
+      public override int GetHashCode() {
+         return ip.GetHashCode() ^ port.GetHashCode();
+      }
+
+      public override bool Equals(object? obj) {
+         if (obj == null) {
+            return false;
+         }
+         RemoteConnectionInfo other = obj as RemoteConnectionInfo;
+         return other != null && other.ip == ip && other.port == port;
+      }
+
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/RemoteMessage.cs

@@ -9,7 +9,7 @@ namespace BCG
    /// </summary>
    public enum RemoteMessageType
    {
-      heartbeat, data, connection, disconnection
+      Heartbeat, Data, Connection, Disconnection
    }
 
    /// <summary>
@@ -45,5 +45,11 @@ namespace BCG
          get { return myConnInfo; }
       }
 
+      override public string ToString()
+      {
+         Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
+         return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
+      }
+
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs

@@ -5,39 +5,30 @@ using System.Text;
 //using BCG.Utilities.General;
 //using BCG.Utilities;
 
-namespace BCG
-{
-   public class ServerToClientManagedTCPConnectionAcceptor
-   {
+namespace BCG {
+   public class ServerToClientManagedTCPConnectionAcceptor {
 
       private static Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor> myPortToSingleServerAcceptorMap = new Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor>();
       private static object mySyncObject = new object();
 
-      public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan)
-      {
+      public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
          SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
          int portNo = theAcceptor.activateWithRandomPort(handler, clientHBTimeSpan);
 
-         lock (mySyncObject)
-         {
+         lock (mySyncObject) {
             myPortToSingleServerAcceptorMap[portNo] = theAcceptor;
 
          }
-         return portNo;         
+         return portNo;
       }
 
-      public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan)
-      {
-         lock (mySyncObject)
-         {
-            if (myPortToSingleServerAcceptorMap.ContainsKey(port))
-            {
+      public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
+         lock (mySyncObject) {
+            if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
                string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
                // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
                return port;
-            }
-            else
-            {
+            } else {
                SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
                int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan);
                myPortToSingleServerAcceptorMap[assignedport] = theAcceptor;
@@ -46,6 +37,23 @@ namespace BCG
          } //end lock
       } //end activate code
 
+      public static int accept_connections(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
+         lock (mySyncObject) {
+            if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
+               string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
+               // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
+               return port;
+            } else {
+               SingleServerToClientManagedTCPConnectionAcceptor ps_acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+               int assignedport = ps_acceptor.accept_on_port(handler, "127.0.0.1", port, clientHBTimeSpan);
+               myPortToSingleServerAcceptorMap[assignedport] = ps_acceptor;
+               return assignedport;
+            }
+         } //end lock
+      } //end activate code
+
+
+
 
    }
 }

src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs

@@ -4,13 +4,12 @@ using System.Text;
 using System.Threading;
 using System.Net;
 using System.Net.Sockets;
+using System.Threading.Tasks;
 
 //using BCG.Utilities;
 
-namespace BCG
-{               
-   public class SingleServerToClientManagedTCPConnectionAcceptor
-   {
+namespace BCG {
+   public class SingleServerToClientManagedTCPConnectionAcceptor {
       private TimeSpan myHBTimeSpan,
                               myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250);
       private ITCPConnectionHandler myHandler;
@@ -18,58 +17,50 @@ namespace BCG
       private int myPort;
 
 
-      public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan)
-      {
+      public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
          myHBTimeSpan = clientHBTimeSpan;
          myHandler = handler;
          myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
          int port = 0;
          bool madeItThrough = false;
-//         BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
+         //         BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
          Random randomizer = new Random();
          PubSubPortRange theRange = PubSub3Config.instance.PS3_PortRange;
 
-        
+
          //if the range of ports is 1000, then trying 5,000 times means 
          //your odds of missing an open port are less than 1%
 
-         int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;         
+         int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;
          int totalAttempts = 0;
-         while (!madeItThrough)
-         {
-            port = randomizer.Next(); 
+         while (!madeItThrough) {
+            port = randomizer.Next();
             port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort));
 
-            try
-            {
+            try {
                myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port));
                madeItThrough = true;
-            }
-            catch (Exception)
-            {
+            } catch (Exception) {
                totalAttempts++;
-               if (totalAttempts < maxAttempts){
+               if (totalAttempts < maxAttempts) {
                   //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection);
-               }
-               else
-               {
+               } else {
                   // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details.");
                }
             }
          }
          myPort = port;
          myAcceptSock.Listen(250);
-         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); 
+         myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
 
          return myPort;
       }
 
-      public int activate (ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan)
-      {
+      public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
          myHBTimeSpan = clientHBTimeSpan;
          myHandler = handler;
          myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-         IPEndPoint endpoint = new IPEndPoint (IPAddress.Any, port);
+         IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port);
          myAcceptSock.Bind(endpoint);
          myAcceptSock.Listen(250);
          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
@@ -77,15 +68,18 @@ namespace BCG
          return myPort;  // not sure why we need this class variable myPort, but I am just keeping it up with the existing code
       }
 
-      private void myOnAccept(IAsyncResult res)
-      {
+      public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) {
+         Task.Run(async () => {
+            await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1));
+         });
+         return port;
+      }
+
+      private void myOnAccept(IAsyncResult res) {
          Socket newSock;
-         try
-         {
+         try {
             newSock = myAcceptSock.EndAccept(res);
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             // BlueEventLog.logEvent(ex);
             Thread.Sleep(myWaitTimeBeforeRetrying);
             myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
@@ -98,10 +92,10 @@ namespace BCG
          ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan);
          myHandler.onConnection(newConn);
          myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
-         string ip;int port;
-         newConn.getPeerAddress (out ip, out port);
+         string ip; int port;
+         newConn.getPeerAddress(out ip, out port);
          string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort);
-//         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
+         //         BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
          //Console.WriteLine(connMsg);       
       }
    }

src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs

@@ -0,0 +1,113 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+using System.Xml.Serialization;
+using System.Reflection;
+using BCG.Utilities.General;
+using System.Net.WebSockets;
+
+namespace BCG
+{
+   // utility functions for TCP socket async communications. Does the following
+   // client to server connect
+   // server to client connect accepting
+   // send data
+   // receive data
+   public static class TCPSocketUtils
+   {
+      #region socket opening and connecting
+      // keeps accepting connections forever
+      public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler,
+                                                            TimeSpan wait_after_error)
+      {
+         IPAddress ipAddress = String.IsNullOrEmpty(ip) ? IPAddress.Parse(ip) : IPAddress.Any;
+         var ip_end_point = new IPEndPoint(ipAddress, port);
+         Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+         listener.Bind(ip_end_point);
+         listener.Listen(10);
+         Socket client_socket = null;
+         while (true)
+         {
+            try
+            {
+               // keeps on accepting connections
+               client_socket = await listener.AcceptAsync();
+               AsyncTCPConnection2 new_conn = new(client_socket, AsyncTCPConnection2.ConnectionSide.Server);
+               handler.onConnection(new_conn);
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
+               Task.Delay(wait_after_error);
+            }
+         }
+      }
+
+      // connects to the server and ona  successful connection, calls con__handler.on_connection_async with an ITCPConnection.
+      // In case of failures to connect it waits for a given amount of time before retrying 
+      public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
+                                                TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024)
+      {
+         Socket client_socket = null;
+         for (int i = 0; i < n_max_attempts; i++)
+         {
+            try
+            {
+               string log_conn = $"{ip}:{server_port}";
+               bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} attempt no: {i}");
+               IPAddress server_ip_address = IPAddress.Parse(ip);
+               client_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+               await client_socket.ConnectAsync(server_ip_address, server_port);
+               bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} successfully connected after {i} attempts");
+               ITCPConnection new_conn = new AsyncTCPConnection2(client_socket, AsyncTCPConnection2.ConnectionSide.Client);
+               conn_handler.onConnection(new_conn);
+               break;
+            }
+            catch (Exception ex)
+            {
+               bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
+               Task.Delay(wait_after_failed_attempt);
+            }
+         }
+         return client_socket;
+      }
+      #endregion
+
+      #region data sending/receiving
+
+      // prefixes the size of the data and sends it as 2 sends, for size and data
+      // (can be done as one send perhaps)
+      public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff)
+      {
+         var data_sz = data_buff.Length;
+         byte[] sz_buff = new byte[4];
+         // aend the size first
+         BinaryWriter bw = new BinaryWriter(new MemoryStream(sz_buff));
+         bw.Write(data_sz);
+         await nw_stream.WriteAsync(sz_buff, 0, 4);
+         await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
+      }
+
+      // reads the bytes on a connected socket, by doing the opposite of the above,
+      // reads the size first and then the data
+      public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream)
+      {
+         byte[] sz_buffer = new byte[4];
+         //Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
+         int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length);
+         BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer));
+         int msg_sz = br.ReadInt32();
+         byte[] data_buffer = new byte[msg_sz];
+         bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length);
+         //Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
+         return data_buffer;
+      }
+      #endregion
+   }
+}

src/DataStreamSuite/PubSub/PubSubMessage.cs

@@ -4,31 +4,39 @@ using System.IO;
 using BCG;
 using BCG.Utilities;
 
-namespace BCG
-{
-   // Tuple of 
-   public class PubSubMessage
-   {
-      public PubSubMessage (MetaData metaData, Stream body)
-      {this.metaData = metaData; this.body = body;}
+namespace BCG {
+   // PubItems are published to the pubsub wrapped in this class. It contains PubMetaData, wrapped in the MetaData 
+   // object and the actualy published payload in the body stream 
+   public class PubSubMessage {
+      public PubSubMessage(MetaData metaData, Stream body) { this.meta_data = metaData; this.body = body; }
 
-      public PubSubMessage (MetaData metaData)
-      {this.metaData = metaData;}
+      public PubSubMessage(PubMetaData pub_meta_data, Stream body)
+      : this(new MetaData(pub_meta_data), body) { }
 
-      public MetaData metaData;
+      public PubSubMessage(MetaData metaData) { this.meta_data = metaData; }
+
+      public MetaData meta_data;
       public Stream body = new MemoryStream(0);
 
-      public Stream toStream()
-      {
+      public Stream to_stream() {
          MemoryStream ms = new MemoryStream();
-         metaData.save(ms);
-         if (body.Length == 0 || body.Length == body.Position)
+         meta_data.save(ms);
+         if (body.Length == 0 || body.Length == body.Position) {
+            bts.Log.Gen.error("Error: Trying to publishing message with no content.");
             Console.WriteLine("Error: Trying to publishing message with no content.");
+         }
          StreamUtilityMethods.copyTo(new Stream[] { body }, ms);
-
          ms.Position = 0;
          return ms;
       }
 
+      public byte[] to_bytes() {
+         return StreamUtilityMethods.getBytesFromStream(to_stream());
+      }
+
+      public override string ToString() {
+         return $"PubSubMessage meta:data: {meta_data.ToString()} body.Posiiton: {body.Position} body.Length: {body.Length}";
+      }
+
    }
 }

src/DataStreamSuite/PubSub/PubSubUpdate.cs

@@ -2,22 +2,21 @@ using System;
 using System.Collections;
 using System.IO;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// this is the data returned from the
    /// pubsub server
    /// </summary>
-   public class PubSubUpdate
-   {
-      
-      public PubSubUpdate(string tag, string tagType, Stream body)
-      { this.tag = tag; this.body = body; this.tagType = tagType; }
+   public class PubSubUpdate {
+
+      public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
 
       public string tag;
       public Stream body;
       public string tagType;
 
-      
+      public override string ToString() {
+         return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
+      }
    };
 }

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -7,171 +7,67 @@ using System.Text.RegularExpressions;
 
 using BCG.Utilities;
 
-namespace BCG
-{
-   public class PublisherCore : IPublisherCore
-   {
-      //private static BlueCommManaged.IPubSub myBCPS;
-      
-      // TODO: what is this?
-      // public static void setPubSub(BlueCommManaged.IPubSub ps)
-      // {
-      //    lock(typeof(PublisherCore))
-      //    {
-      //       BlueEventLog2.instance().assert(myBCPS == null, 
-      //          "PublisherCore.setPubSub() should be called only once and before any use of PublisherCore");
-      //       myBCPS = ps;
-      //    }
-      // }
+namespace BCG {
+   public class PublisherCore : IPublisherCore {
 
       public static string pubSubBlueCommPrefix = "ps/";
       public static string pubSubBlueCommTransactionPrefix = "tgps/";
       public static string pubSubBlueCommTransactionComponentPrefix = "tcps/";
-      // public static BlueCommManaged.IPubSub getPubSub()
-      // {
-      //    lock (typeof(PublisherCore)) {
-      //       return myBCPS;
-      //    }
-      // }
 
-      enum Destination
-      {
+      enum Destination {
          None,
          BlueComm,
          PubSub,
          Both
       }
 
-      //private IPublisherCore myBlueCommPublisher = null;
-      private IPublisherCore myPubSubPublisher = null;
-      private Dictionary<string, Destination> myPubDestinations =
-         new Dictionary<string, Destination>();     
+      private IPublisherCore _publisher_core = null;
 
-      public PublisherCore()
-      {
-         myPubSubPublisher = new StaticPublisherCore();
-         //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
+      public PublisherCore() {
+         _publisher_core = new StaticPublisherCore();
       }
 
-      public PublisherCore(IPubSubRouter theRouter)
-      {
-         myPubSubPublisher = new StaticPublisherCore(theRouter);
-         //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
+      public PublisherCore(IPubSubRouter theRouter) {
+         _publisher_core = new StaticPublisherCore(theRouter);
       }
 
-      public PublisherCore(string URL)
-      {
-         myPubSubPublisher = new StaticPublisherCore(URL);
-         //myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
+      public PublisherCore(string URL) {
+         _publisher_core = new StaticPublisherCore(URL);
       }
 
       #region IPublisherCore Methods
 
-      public void Dispose()
-      {
-         myPubSubPublisher.Dispose();
-         //myBlueCommPublisher.Dispose();
+      public void Dispose() {
+         _publisher_core.Dispose();
       }
-      public int getNumberOfWaitingMessages() { return myPubSubPublisher.getNumberOfWaitingMessages(); }
+      public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
 
-      private Destination myGetPublishDestionation (string tag)
-      {
-         return Destination.PubSub;
-         // lock (this)
-         // {
-         //    if (PubSub3Config.instance.BlueComm_mustDie)
-         //       return Destination.PubSub;
-         //    else
-         //    {
-         //       Destination d;
-         //       return Destination.PubSub;
-         //       if (myPubDestinations.TryGetValue(tag, out d))
-         //          return d;
-         //       else
-         //       {
-         //          foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInPubSub)
-         //             if (Regex.IsMatch(tag, regexp))
-         //             {
-         //                d = Destination.PubSub;
-         //                foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInPubSub)
-         //                   if (Regex.IsMatch(tag, regexp2))
-         //                   {
-         //                      d = Destination.None;
-         //                      break;
-         //                   }
-         //                break;
-         //             }
-
-         //          if (!tag.StartsWith("mon"))
-         //             foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInBlueComm)
-         //                if (Regex.IsMatch(tag, regexp))
-         //                {
-         //                   if (d == Destination.PubSub)
-         //                      d = Destination.Both;
-         //                   else
-         //                      d = Destination.BlueComm;
-         //                   foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInBlueComm)
-         //                      if (Regex.IsMatch(tag, regexp2))
-         //                      {
-         //                         if (d == Destination.Both)
-         //                            d = Destination.PubSub;
-         //                         else if (d == Destination.BlueComm)
-         //                            d = Destination.None;
-         //                         break;
-         //                      }
-         //                   break;
-         //                }
-
-         //          myPubDestinations[tag] = d;
-         //          return d;
-         //       }
-         //    }
-         // }
-      }
-
-      public void publish(string tag, Stream content)
-      {
+      public void publish(string tag, Stream content) {
          publish(tag, "", content);
       }
 
-      public void publish(string tagType, string tag, Stream content)
-      {
-         Destination d = myGetPublishDestionation(tag);
-         if (d == Destination.PubSub || d == Destination.Both)
-            myPubSubPublisher.publish(tagType, tag, content);
-         if (d == Destination.BlueComm || d == Destination.Both){}
-            //myBlueCommPublisher.publish(tagType, tag, content);
+      public void publish(string tagType, string tag, Stream content) {
+         bts.Log.Gen.info($"PublisherCore.publish tag: {tag} stream.Length: {content.Length} stream.Position: {content.Position}");
+         _publisher_core.publish(tagType, tag, content);
       }
 
-      public void beginTransaction()
-      {
-         myPubSubPublisher.beginTransaction();
-         //myBlueCommPublisher.beginTransaction();
+      public void beginTransaction() {
+         _publisher_core.beginTransaction();
       }
 
-      public void commitTransaction()
-      {
-         myPubSubPublisher.commitTransaction();
-         //myBlueCommPublisher.commitTransaction();
+      public void commitTransaction() {
+         _publisher_core.commitTransaction();
       }
 
-      public void abortTransaction()
-      {
-         myPubSubPublisher.abortTransaction();
-         //myBlueCommPublisher.abortTransaction();
+      public void abortTransaction() {
+         _publisher_core.abortTransaction();
       }
 
-      public void clearState(string tagToClear)
-      {
-         Destination d = myGetPublishDestionation(tagToClear);
-         if (d == Destination.PubSub || d == Destination.Both)
-            myPubSubPublisher.clearState(tagToClear);
-         if (d == Destination.BlueComm || d == Destination.Both){}
-            //myBlueCommPublisher.clearState(tagToClear);
+      public void clearState(string tagToClear) {
+         _publisher_core.clearState(tagToClear);
       }
 
-      public bool inTransaction
-      { get { return myPubSubPublisher.inTransaction; } }
+      public bool inTransaction { get { return _publisher_core.inTransaction; } }
 
       #endregion
 

src/DataStreamSuite/PubSub/PubsubDeserializationController.cs

@@ -3,27 +3,25 @@ using System.IO;
 using System.Collections;
 using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Controls (ie creates dynamically) an IPubsubDeserializer based on the type name which is 
    /// part of the published data. Since its also a IPubsubDeserializer, all subsequent calls
    /// are passed though to the interface
    /// </summary>
-   public class PubsubDeserializationController : IPubsubDeserializer
-   {
+   public class PubsubDeserializationController : IPubsubDeserializer {
       private IPubsubDeserializer myDeserializer;
 
       public PubsubDeserializationController() { }
 
-      public void update(Stream stream)
-      {
+      public void update(Stream stream) {
+         bts.Log.Gen.info("PubsubDeserializationController.update stream: {stream.Position} / {stream.Length}");
          byte[] ops;
          Array rows;
          deserialize(new Stream[] { stream }, out ops, out rows);
       }
-      public void update(byte[] bytes)
-      {
+      public void update(byte[] bytes) {
+         bts.Log.Gen.info("PubsubDeserializationController.update bytes: {bytes.Length}");
          byte[] ops;
          Array rows;
          deserialize(new Stream[] { new MemoryStream(bytes, 0, bytes.Length, false, true) }, out ops, out rows);
@@ -31,39 +29,36 @@ namespace BCG
 
       #region IPubsubDeserializer Members
 
-      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         if (myDeserializer != null)
-         {
+      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
+
+         if (myDeserializer != null) {
             myDeserializer.deserialize(streams, out theOps, out theRows);
-         }
-         else
-         {
+            bts.Log.Gen.info($"PubsubDeserializationController.deserialize myDeserializer: {myDeserializer} ops: {theOps.Length} theRows: {theRows.Length}");
+         } else {
             Stream totalStream = UtilityMethods.getAggregatedStream(streams);
             long initialPos = totalStream.Position;
             BinaryReader br = new BinaryReader(totalStream);
             byte opCode = br.ReadByte();
-            if (opCode == StreamOpCode.create)
-            {
+            bts.Log.Gen.info($"PubsubDeserializationController.deserialize opCode: {opCode}");
+            if (opCode == StreamOpCode.create) {
                string deserName = br.ReadString();
-               if (deserName == "BCG.PubsubUniversalDeserializer") // so we can play old replay files which would not include the assembly name
-                  deserName = "BCG.PubsubUniversalDeserializer, DataStreamGateway";
-               myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
-                  as IPubsubDeserializer;
+               // TODOds
+               if (deserName == "BCG.PubsubUniversalDeserializer, DataStreamGateway") // so we can play old replay files which would not include the assembly name
+                  deserName = "BCG.PubsubUniversalDeserializer, DataStreamCore";
+               myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) as IPubsubDeserializer;
                totalStream.Position = initialPos;
                myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
-            }
-            else
-            {
-               // BlueEventLog.logEvent("Received a non create before a create. Bad op code was " + opCode.ToString(), BlueEventLevel.Warning);
+               bts.Log.Gen.info($"PubsubDeserializationController.deserialize myDeserializer: {myDeserializer} ops: {theOps.Length} theRows: {theRows.Length}");
+
+            } else {
+               bts.Log.Gen.error($"Received a non create before a create. Bad op code was : {opCode} ");
                theOps = new byte[] { };
                theRows = new object[] { };
             }
          }
       }
 
-      public byte[] getState()
-      {
+      public byte[] getState() {
          return myDeserializer == null ? new byte[] { } : myDeserializer.getState();
       }
 

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -9,17 +9,14 @@ using System.IO;
 using System.Diagnostics;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
 
    public class ManagedTCPMessageHub : IPubsubMessageHub,
                                        ITCPConnectionHandler,
-                                       ITCPMessageHandler
-   {
+                                       ITCPMessageHandler {
       private int myPort;
       private bool myIsDisposed = false;
-      public int port
-      {
+      public int port {
          get { return myPort; }
          //property is read only
 
@@ -66,8 +63,7 @@ namespace BCG
       private DateTime myLastHeartbeatDeadCollectionTime = DateTime.Now;
       private DateTime myLastDisposedDeadCollectionTime = DateTime.Now;
 
-      public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod)
-      {
+      public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod) {
          myPort = port;
          myHeartbeatSendInterval = hbSendInterval;
          mySubscriberDropThreshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hbSendInterval.TotalMilliseconds * missingHBsForGC));
@@ -81,24 +77,21 @@ namespace BCG
 
       #region IPubsubMessageHub Members
 
-      public void registerDataCallback(DataDelegate del)
-      {
+      public void registerDataCallback(DataDelegate del) {
          // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
          myDataCb = del;
       }
 
-      public void registerConnectionCallback(ConnectionStatusChange del)
-      {
+      public void registerConnectionCallback(ConnectionStatusChange del) {
          //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
          myConnCb = del;
       }
 
-      public void activate()
-      {
+      public void activate() {
          if (PubSub3Config.instance.PS3_BindToRandomPort || myPort == 0)
             myPort = ServerToClientManagedTCPConnectionAcceptor.activateWithRandomPort(this, myHeartbeatSendInterval);
          else
-            myPort = ServerToClientManagedTCPConnectionAcceptor.activate(this, myPort, myHeartbeatSendInterval);
+            myPort = ServerToClientManagedTCPConnectionAcceptor.accept_connections(this, myPort, myHeartbeatSendInterval);
 
          string msg = String.Format("TCP Component is ready, listening on port {0} for incoming connections", myPort);
          Console.WriteLine(msg);
@@ -108,19 +101,16 @@ namespace BCG
          doItAllThread.Start();
       }
 
-      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements)
-      {
+      public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) {
          return myReceivedData.dequeueMultipleElements(timeout, maxElements);
       }
 
-      public bool send(string receiver, byte[] data)
-      {
+      public bool send(string receiver, byte[] data) {
          mySendData.enqueue(new Pair<string, byte[]>(receiver, data));
          return true;
       }
 
-      public bool send(string receiver, System.IO.Stream str)
-      {
+      public bool send(string receiver, System.IO.Stream str) {
          BinaryReader br = new BinaryReader(str);
          return send(receiver, br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
       }
@@ -133,17 +123,13 @@ namespace BCG
       /// Reports back connected and disconnected connections
       /// via the ConnectionStatucChange callback
       /// </summary>
-      public void doWork()
-      {
-         if (myConnCb != null)
-         {
-            foreach (string nc in myNewConnectionURLs.dequeueAll())
-            {
+      public void doWork() {
+         if (myConnCb != null) {
+            foreach (string nc in myNewConnectionURLs.dequeueAll()) {
                //BlueEventLog.logEvent(String.Format("doWork : New client {0} connected, making connection callback", nc), PS3LogLevels.connection);
                myConnCb(nc, true);
             }
-            foreach (string dc in myDeadConnectionURLs.dequeueAll())
-            {
+            foreach (string dc in myDeadConnectionURLs.dequeueAll()) {
                //BlueEventLog.logEvent(String.Format("doWork : Client {0} disconnected, making disconnection callback", dc), PS3LogLevels.disconnection);
                myConnCb(dc, false);
             }
@@ -160,8 +146,7 @@ namespace BCG
       /// stored in the hashtable and used for later communication)
       /// </summary>
       /// <param name="connection"></param>
-      public void onConnection(ITCPConnection connection)
-      {
+      public void onConnection(ITCPConnection connection) {
          myNewConnections.enqueue(connection);
       }
 
@@ -169,48 +154,38 @@ namespace BCG
 
       #region ITCPMessageHandler Members
 
-      public void onData(RemoteMessage message)
-      {
+      public void onData(RemoteMessage message) {
          Console.WriteLine("ManagedTCPMessgageHub.onData recv msg :" + message.remoteConnectionInfo.ToString());
          string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString();
-         if (message.messageType == RemoteMessageType.data)
-         {
+         if (message.messageType == RemoteMessageType.Data) {
             if (myDataCb != null)
                myDataCb(sender, message.message);
             else
                myReceivedData.enqueue(new Pair<string, byte[]>(sender, message.message));
-         }
-         else if (message.messageType == RemoteMessageType.heartbeat)
+         } else if (message.messageType == RemoteMessageType.Heartbeat)
             myReceivedHeartbeats.enqueue(sender);
-         else
-         {
+         else {
             //BlueEventLog.logEvent("Unknown message type from the client", BlueEventLevel.Error);
          }
       }
 
       #endregion
 
-      class TimmingInfo
-      {
-         public TimmingInfo()
-         {
+      class TimmingInfo {
+         public TimmingInfo() {
             myTimer = new Stopwatch();
             reset();
          }
 
-         public void start()
-         {
-            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0)
-            {
+         public void start() {
+            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) {
                myTimer.Reset();
                myTimer.Start();
             }
          }
 
-         public void stop()
-         {
-            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0)
-            {
+         public void stop() {
+            if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) {
                myTimer.Stop();
                myMin = Math.Min(myMin, myTimer.ElapsedMilliseconds);
                myMax = Math.Max(myMax, myTimer.ElapsedMilliseconds);
@@ -219,16 +194,14 @@ namespace BCG
             }
          }
 
-         public void reset()
-         {
+         public void reset() {
             myMin = Double.MaxValue;
             myMax = Double.MinValue;
             myTotal = 0;
             myNMeasurements = 0;
          }
 
-         public override string ToString()
-         {
+         public override string ToString() {
             if (myNMeasurements > 0)
                return String.Format("min={0} max={1} avg={2} nMeasurements={3}", myMin, myMax, myTotal / myNMeasurements, myNMeasurements);
             else
@@ -242,8 +215,7 @@ namespace BCG
          private double myTotal;
       }
 
-      private void myDoItAll()
-      {
+      private void myDoItAll() {
          Dictionary<string, TimmingInfo> times = new Dictionary<string, TimmingInfo>();
          times.Add("myHandleNewConnections", new TimmingInfo());
          times.Add("myCollectDisposedConnections", new TimmingInfo());
@@ -252,12 +224,9 @@ namespace BCG
 
          DateTime lastTimmingLog = DateTime.Now;
 
-         while (true)
-         {
-            try
-            {
-               while (true)
-               {
+         while (true) {
+            try {
+               while (true) {
                   ///new connections
                   times["myHandleNewConnections"].start();
                   myHandleNewConnections();
@@ -279,16 +248,12 @@ namespace BCG
                   Pair<string, byte[]> p = mySendData.dequeue(myReadSendQueueTimeout);
 
                   times["send"].start();
-                  if (p != null)
-                  {
+                  if (p != null) {
                      ITCPConnection conn;
                      myConnections.TryGetValue(p.first, out conn);
-                     if (conn != null)
-                     {
+                     if (conn != null) {
                         conn.send(p.second);
-                     }
-                     else
-                     {
+                     } else {
                         string warningMsg = String.Format("sending data to {0} failed, this client might have " +
                            "disconnected, in which case it will be garbage collected within 60 secs", p.first);
                         //BlueEventLog.logEvent(warningMsg, PS3LogLevels.disconnection);
@@ -297,25 +262,21 @@ namespace BCG
                   times["send"].stop();
 
                   if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0 &&
-                     (DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs)
-                  {
+                     (DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs) {
                      lastTimmingLog = DateTime.Now;
 
                      StringBuilder b = new StringBuilder();
                      b.AppendFormat("myDoItAll stats:\n");
                      b.AppendFormat("queue size: {0}\n", mySendData.getCount());
 
-                     foreach (string timerName in times.Keys)
-                     {
+                     foreach (string timerName in times.Keys) {
                         b.AppendFormat("{0}: {1}\n", timerName, times[timerName]);
                         times[timerName].reset();
                      }
                      //BlueEventLog.logEvent(b.ToString(), PS3LogLevels.performance);
                   }
                }
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
                //BlueEventLog.logEvent(ex);
                if (ex is OutOfMemoryException)
                   throw ex;
@@ -323,19 +284,15 @@ namespace BCG
          }
       }
 
-      private void myCollectDisposedConnections()
-      {
-         if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod)
-         {
+      private void myCollectDisposedConnections() {
+         if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod) {
             myLastDisposedDeadCollectionTime = DateTime.Now;
             List<string> disposedURLs = new List<string>();
-            foreach (KeyValuePair<string, ITCPConnection> p in myConnections)
-            {
+            foreach (KeyValuePair<string, ITCPConnection> p in myConnections) {
                if (p.Value.isDisposed)
                   disposedURLs.Add(p.Key);
             }
-            foreach (string url in disposedURLs)
-            {
+            foreach (string url in disposedURLs) {
                string disposeMsg = String.Format("Collecting disposed connections : {0} is disposed by the connection itself, " +
                   "connection count before deleting this = {1}", url, myConnections.Count);
                Console.WriteLine(disposeMsg);
@@ -346,10 +303,8 @@ namespace BCG
          }
       }
 
-      private void myCollectMissingHeartbeatConnections()
-      {
-         if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold)
-         {
+      private void myCollectMissingHeartbeatConnections() {
+         if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) {
             myLastHeartbeatDeadCollectionTime = DateTime.Now;
             ///these are all the hbs received bw the last
             ///and current collectDeadConnection calls
@@ -365,10 +320,9 @@ namespace BCG
             foreach (string clientURL in myConnections.Keys)
                if (!indexedHbs.ContainsKey(clientURL))
                   deadClients.Add(clientURL);
-            foreach (string deadClient in deadClients)
-            {
+            foreach (string deadClient in deadClients) {
                string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
-                                                   "connection count before deleting this = {1}", deadClient, myConnections.Count);                                                   
+                                                   "connection count before deleting this = {1}", deadClient, myConnections.Count);
                //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection);
                Console.WriteLine(disposeMsg);
                myConnections[deadClient].Dispose();
@@ -378,11 +332,9 @@ namespace BCG
          }
       }
 
-      private void myHandleNewConnections()
-      {
+      private void myHandleNewConnections() {
          IList<ITCPConnection> newConns = myNewConnections.dequeueAll();
-         foreach (ITCPConnection conn in newConns)
-         {
+         foreach (ITCPConnection conn in newConns) {
             string ip;
             int port;
             conn.getPeerAddress(out ip, out port);
@@ -399,13 +351,11 @@ namespace BCG
       }
 
       #region IDisposable Memebers
-      public void Dispose()
-      {
+      public void Dispose() {
          myIsDisposed = true;
 
       }
-      public bool isDisposed
-      {
+      public bool isDisposed {
          get { return myIsDisposed; }
       }
       #endregion

src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs

@@ -4,23 +4,20 @@ using System.Text;
 using BCG.Utilities.General;
 using System.IO;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
    /// A subscriber can subscribe to it, and find out of a PubSub is alive or 
    /// not. Infact more so, if the subsriptions are working or not.
    /// </summary>
-   internal class SystemHeartbeatPublisher : IWorker
-   {
+   internal class SystemHeartbeatPublisher : IWorker {
       public static string tagType = "System_Heartbeat";
       public static string tagName = "System_Heartbeat";
-     
+
       private static SystemHeartbeatPublisher mySelf;
 
-      private SystemHeartbeatPublisher ()
-      {
-         
+      private SystemHeartbeatPublisher() {
+
          ///this is using a multithreaded MasterWorker since the 
          ///regular one is called within the MessageReactor, which
          ///waits 2500 msecs before calling doWork. This 2500 msecs
@@ -29,16 +26,14 @@ namespace BCG
          MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500));
       }
 
-      public static void init()
-      {
+      public static void init() {
          if (mySelf == null)
             mySelf = new SystemHeartbeatPublisher();
       }
 
       #region IWorker Members
 
-      public void doWork()
-      {
+      public void doWork() {
          PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString);
          MetaData md = new MetaData(pub);
          MemoryStream ms = new MemoryStream();

src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs

@@ -16,10 +16,8 @@ using System.Collections.Generic;
 using mbts;
 using mbts.spark;
 
-namespace BCG
-{
-   public class PubSubMessageReactor
-   {
+namespace BCG {
+   public class PubSubMessageReactor {
       private ServerMessageHub myMessageHub;
       private delegate void MyMsgHandler(string sender, Stream msg, MetaData m);
       private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>();
@@ -39,32 +37,24 @@ namespace BCG
       /* This delegate allows pubsub components to be notified when the identity of the pubsub is known*/
       public delegate void IdentityFoundDelegate(string identity, bool isRoot);
       public event IdentityFoundDelegate onIdentityFound;
-      public void fireIdentityFound(string identity, bool isRoot)
-      {
+      public void fireIdentityFound(string identity, bool isRoot) {
          if (onIdentityFound != null)
             onIdentityFound(identity, isRoot);
       }
 
-      private IPubSubGovernor myGovernor;
-
-
       public Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>();
 
       public PubSubMessageReactor() : this(false, false, string.Empty, string.Empty) { }
       public PubSubMessageReactor(string q) : this(false, false, string.Empty, q) { }
 
-      public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq)
-      {
+      public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq) {
          myOperationToHandler.Add(typeof(PubMetaData).Name, new MyMsgHandler(myHandlePub));
          myOperationToHandler.Add(typeof(SubMetaData).Name, new MyMsgHandler(myHandleSub));
          myOperationToHandler.Add(typeof(UnSubMetaData).Name, new MyMsgHandler(myHandleUnsub));
          myOperationToHandler.Add(typeof(TransactionMetaData).Name, new MyMsgHandler(myHandleTransaction));
 
-         // TODO, governor needs to be removed since its subststaion retired code
-         // if (PubSub3Config.instance.PS3_Components.governor != null)
-         //    myGovernor = PubSubComponentFactory.buildGovernor(this);
 
-         //BlueEventLog.logEvent("System Initialized", PS3LogLevels.info);
+         bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
 
          myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
          myMessageHub = ServerMessageHub.instance;
@@ -75,93 +65,49 @@ namespace BCG
 
       #region message handlers
 
-      public PubSubState state
-      {
+      public PubSubState state {
          get { return myState; }
          //this variable is read only
 
       }
 
-      private void myHandlePub(string sender, Stream msg, MetaData m)
-      {
-
+      // all publishes to the pubsub end here
+      private void myHandlePub(string sender, Stream msg_str, MetaData meta_data) {
+         bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub sender: {sender} metadata: {meta_data} str.pos: {msg_str.Position} str.len: {msg_str.Length}");
          mySingleTagTimeSliceTimer.start();
          ///the position in the stream is set to where the data starts with,
          ///since the meta data was already read in the myProcessMsg
          PubMetaData pub = new PubMetaData();
-         m.copyTo(pub);
+         meta_data.copyTo(pub);
          pub.connection = sender;
-         Console.WriteLine("PubSubMessageReactor.myHandlePub sender: " + sender + " tag: " + pub.tag);
-
-         //check to see if this publish took a while to get here
          pub.warnIfLatencyExceedsThreshold();
-
-         //check the governor to see if this publish is allowed, if we have a governor
-         PubSubGovernorResponse response = PubSubGovernorResponse.Accepted;
-         if (myGovernor != null)
-            response = myGovernor.requestToPublish(pub, msg);
-
-         if (response != PubSubGovernorResponse.Accepted)
-         {
-            // BlueEventLog2.instance().logEvent(string.Concat("PubSub Governor ",
-            //                                                 myGovernor.GetType().Name,
-            //                                                 " returned response ",
-            //                                                 response.ToString(),
-            //                                                 " publish on tag",
-            //                                                 pub.tag,
-            //                                                 "was ignored."), BlueEventLog2Level.Warning);
-            return;
-         }
-
-
-         //report the fact that we have a publish on this tag, if it's a debug tag
-         if (PubSub3Config.instance.isTagDebugTag(pub.tag))
-         {
-            // BlueEventLog.logEvent(string.Concat(GetType().Name,
-            //                                     " got publish on tag ",
-            //                                     pub.tag,
-            //                                     " from app ", pub.publishingApp,
-            //                                     " over connection ",
-            //                                     pub.connection, ".\n",
-            //                                     pub.isCompressed ? "Data was compressed.\n" : "Data was not compresed.\n",
-            //                                     "total size: ",
-            //                                     pub.size),
-
-            //                                     PS3LogLevels.debug);
-
-
-         }
+         var subs_for_tag = myState.getSubscriberConnectionsForTag(pub.tag);
+         bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub {subs_for_tag.Count} subs for tag: {pub.tag}");
 
          ///for all the subscribers, sends the messages over straightaway
-         foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag))
-         {
+         foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag)) {
             ///the position is adjusted to zero, so as to include
             ///the meta data as part of the message
-            msg.Position = 0;
-            myMessageHub.send(address, msg);
+            msg_str.Position = 0;
+            bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub sending tag: {pub.tag} to {address} len: {msg_str.Length - msg_str.Position}");
+            myMessageHub.send(address, msg_str);
          }
 
-
-
          //reset the position of the message so we can decompress it appropriately
-         msg.Position = 0;
-         msg.Position = m.size();
+         msg_str.Position = 0;
+         msg_str.Position = meta_data.size();
 
-         if (pub.isCompressed)
-         {
-            long mdLength = msg.Position;
-            long compressedSize = msg.Length - mdLength;
-            if (compressedSize == 0)
-            {
+         if (pub.isCompressed) {
+            long mdLength = msg_str.Position;
+            long compressedSize = msg_str.Length - mdLength;
+            if (compressedSize == 0) {
                string errorMsg = "Error: tag " + pub.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
-               //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
+               bts.Log.Gen.error(errorMsg);
                return;
             }
-            msg = StreamDecompressor.decompressStream(msg);
-            long decompressedSize = msg.Length - mdLength;
-
-            if (decompressedSize != pub.decompressedSize || compressedSize != pub.size)
-            {
+            msg_str = StreamDecompressor.decompressStream(msg_str);
+            long decompressedSize = msg_str.Length - mdLength;
+            if (decompressedSize != pub.decompressedSize || compressedSize != pub.size) {
                string compressionWarning = string.Format("Warning: Stream for tag {0} decompressed from {1} to {2} bytes, with {3} byes of metadata. Metadata expected {4} to {5}. Tag state not updated.",
                                                                                                                   pub.tag,
                                                                                                                   compressedSize,
@@ -169,43 +115,30 @@ namespace BCG
                                                                                                                   mdLength,
                                                                                                                   pub.size,
                                                                                                                   pub.decompressedSize);
-
-               //BlueEventLog.logEvent(compressionWarning, BlueEventLevel.Error);
+               bts.Log.Gen.warn(compressionWarning);
                return;
-
             }
-
          }
 
          //give the user a notification of the compression status of the tag has changed. 
          if ((!(myTagCompressionState.ContainsKey(pub.tag)) ||
-                myTagCompressionState[pub.tag] != pub.isCompressed))
-         {
+                myTagCompressionState[pub.tag] != pub.isCompressed)) {
             string compressionMessage;
             myTagCompressionState[pub.tag] = pub.isCompressed;
 
-            if (pub.isCompressed)
-            {
+            if (pub.isCompressed) {
                compressionMessage = "Tag " + pub.tag + " is compressed.";
                //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio);
-            }
-            else
-            {
+            } else {
                compressionMessage = "Tag " + pub.tag + " is not compressed.";
                //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio);
-
             }
-
          }
-
-
          ///updating state
-         msg.Position = 0;
-         msg.Position += m.size();
-         myState.updatePubData(pub, msg);
-
+         msg_str.Position = 0;
+         msg_str.Position += meta_data.size();
+         myState.updatePubData(pub, msg_str);
          mySingleTagTimeSliceTimer.stop();
-
          double thisTagProcessingTime = mySingleTagTimeSliceTimer.lastIntervalTime;
          if (onSingleTagPub != null)
             onSingleTagPub(pub.tag, thisTagProcessingTime);
@@ -213,8 +146,8 @@ namespace BCG
 
       }
 
-      private void myHandleSub(string sender, Stream msg, MetaData m)
-      {
+      private void myHandleSub(string sender, Stream msg, MetaData m) {
+         bts.Log.Gen.info($"PubSubMessageReactor.myHandleSub sender: {sender} metadata: {m} str.pos: {msg.Position} str.len: {msg.Length}");
          SubMetaData sub = new SubMetaData();
          m.copyTo(sub);
          sub.connection = sender;
@@ -222,12 +155,10 @@ namespace BCG
          ///update the state
          myState.sub(sub);
          ///to all these new subscriber s, send the existing data
-         foreach (string tag in sub.tags)
-         {
+         foreach (string tag in sub.tags) {
             byte[] data = myState.getContent(tag);
             //ie data has been published once
-            if (data != null)
-            {
+            if (data != null) {
                string tagType = myState.getTagType(tag);
                ///passing "" as the 4th arg, as PubMeta is used both by the
                ///publisher to publish data to the server (where the publisher
@@ -237,17 +168,14 @@ namespace BCG
                ///needed as of now, so its left black to save a few bytes
                PubMetaData pub;
                MemoryStream realContent;
-               if (PubSub3Config.instance.isTagCompressed(tag))
-               {
+               if (PubSub3Config.instance.isTagCompressed(tag)) {
                   SharpZipCompressor Kompessor = new SharpZipCompressor();
                   realContent = new MemoryStream();
                   Kompessor.compress(new MemoryStream(data), realContent);
 
                   pub = new PubMetaData(tagType, tag, (int)realContent.Length, (int)data.Length, "", "");
 
-               }
-               else
-               {
+               } else {
                   realContent = new MemoryStream(data);
                   pub = new PubMetaData(tagType, tag, data.Length, "", "");
                }
@@ -262,8 +190,7 @@ namespace BCG
          }
       }
 
-      private void myHandleUnsub(string sender, Stream msg, MetaData m)
-      {
+      private void myHandleUnsub(string sender, Stream msg, MetaData m) {
          UnSubMetaData unsub = new UnSubMetaData();
          m.copyTo(unsub);
          if (unsub.connection.Trim().Length == 0)
@@ -271,8 +198,7 @@ namespace BCG
          myState.unsub(unsub);
       }
 
-      private void myHandleTransaction(string sender, Stream msg, MetaData md)
-      {
+      private void myHandleTransaction(string sender, Stream msg, MetaData md) {
          mySingleTagTimeSliceTimer.start();
          TransactionMetaData t = new TransactionMetaData();
          md.copyTo(t);
@@ -280,8 +206,7 @@ namespace BCG
          ///form the structure tag->byte[]*
          Dictionary<string, byte[]> tagToMessages = new Dictionary<string, byte[]>();
          BinaryReader br = new BinaryReader(msg);
-         for (int i = 0; i < nMessages; i++)
-         {
+         for (int i = 0; i < nMessages; i++) {
             MetaData pubMd = new MetaData(msg);
             PubMetaData pub = new PubMetaData();
             pubMd.copyTo(pub);
@@ -289,8 +214,7 @@ namespace BCG
             byte[] tagData = new byte[pub.size];
             msg.Read(tagData, 0, tagData.Length);
             //report the fact that we have a publish on this tag, if it's a debug tag
-            if (PubSub3Config.instance.isTagDebugTag(pub.tag))
-            {
+            if (PubSub3Config.instance.isTagDebugTag(pub.tag)) {
                // BlueEventLog.logEvent(string.Concat(GetType().Name,
                //                                     " got publish on tag ",
                //                                     pub.tag,
@@ -315,13 +239,11 @@ namespace BCG
          }
          ///this gives subscriber->tag*
          Dictionary<string, List<string>> subscriberToTags = myState.getSubscriberConnectionsForTags(tagToMessages.Keys);
-         foreach (KeyValuePair<string, List<string>> kvp in subscriberToTags)
-         {
+         foreach (KeyValuePair<string, List<string>> kvp in subscriberToTags) {
             MemoryStream subStream = new MemoryStream();
             TransactionMetaData subTransaction = new TransactionMetaData(kvp.Value.Count);
             new MetaData(subTransaction).save(subStream);
-            foreach (string tag in kvp.Value)
-            {
+            foreach (string tag in kvp.Value) {
                byte[] tagData = tagToMessages[tag];
                PubMetaData pub = new PubMetaData(myState.getTagType(tag), tag, tagData.Length, "", "");
                new MetaData(pub).save(subStream);
@@ -336,8 +258,7 @@ namespace BCG
          double timeExpended = mySingleTagTimeSliceTimer.lastIntervalTime;
          //Don't try to divide by zero
          int numTags = tagToMessages.Keys.Count;
-         if (numTags != 0)
-         {
+         if (numTags != 0) {
             double timeExpendedPerTag = timeExpended / tagToMessages.Keys.Count;
             if (onSingleTagPub != null)
                foreach (string tag in tagToMessages.Keys)
@@ -351,18 +272,16 @@ namespace BCG
       /// </summary>
       /// <param name="msg"></param>
       /// <returns></returns>
-      private void myProcessMsg(string sender, Stream msg, out MetaData md)
-      {
-         md = new MetaData(msg);
+      private void myProcessMsg(string sender, Stream msg, out MetaData md) {
+         md = new MetaData(msg); // msg's Posiiton moved to end of meta data
          MyMsgHandler op = myOperationToHandler[md["op-code"]];
-         //BlueEventLog.logEvent(string.Format("process message {0}", m));
+         bts.Log.Gen.info($"PubSubMessageReactor.myProcessMsg sender: {sender} metadata: {md}");
          op(sender, msg, md);
       }
 
       #endregion
 
-      private void myHandleConnections(string client, bool connected)
-      {
+      private void myHandleConnections(string client, bool connected) {
          Console.WriteLine("PubSubMessageReactor.myHandleConnections client: " + client + " connected: " + connected);
          ///interested in disconnections only, as connections of no
          ///significance until they subscribe, in which case an entry
@@ -371,13 +290,11 @@ namespace BCG
             myState.removeConnection(client);
       }
 
-      public void stop()
-      {
+      public void stop() {
          myStop = true;
       }
 
-      public void start()
-      {
+      public void start() {
          myReceivingProcThread.Start();
       }
 
@@ -385,8 +302,7 @@ namespace BCG
       /// loops around and reads messages from the messageReceiver. calls processMsg
       /// passing in the IClientMessage. also logs stuff about the message.
       /// </summary>
-      private void myReceivingThreadProc()
-      {
+      private void myReceivingThreadProc() {
          TimeSpan receiveTimeout = new TimeSpan(0, 0, 0, 0, 2500);
          myMessageHub.activate();
 
@@ -398,31 +314,25 @@ namespace BCG
          string sender;
          Stream msg;
          MetaData meta;
-         while (!myStop)
-         {
+         while (!myStop) {
             sender = null;
             msg = null;
             meta = null;
-            try
-            {
+            try {
                List<Pair<string, byte[]>> receivedData = myMessageHub.getUpdates(receiveTimeout, 1);
-               if (receivedData.Count > 0)
-               {
+               if (receivedData.Count > 0) {
                   sender = receivedData[0].first;
                   msg = new MemoryStream(receivedData[0].second);
                   myTimer.start();
                   myProcessMsg(sender, msg, out meta);
                   myTimer.stop();
-                  if (myTimer.lastIntervalTime > myProcessingWarningThreshold)
-                  {
+                  if (myTimer.lastIntervalTime > myProcessingWarningThreshold) {
                      string warningMsg = String.Format("processing metadata {0} took {1} msecs", meta, myTimer.lastIntervalTime);
                      //BlueEventLog.logEvent(warningMsg, PS3LogLevels.performance);
                   }
                }
                myMasterWorker.doWork();
-            }
-            catch (Exception e)
-            {
+            } catch (Exception e) {
                //BlueEventLog.logEvent(e, BlueEventLevel.Error);
                Console.WriteLine("{0} : exception handling message: {1}\n", DateTime.Now, e.Message);
                string errMsg = String.Format("Following information about the meta data and sender is available : " +

src/DataStreamSuite/PubSub/Server/PubSubState.cs

@@ -7,21 +7,19 @@ using System.IO;
 
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Mantains all the state that there is in the server.
    /// This information is keyed off the tags (rather than 
    /// keyed off subscribers or publishers)
-   /// State information consists of 2 distints sets :
+   /// State information consists of 2 distinct sets :
    /// myTagToData consists of all the data published to 
    /// this pubsub.
    /// myTagToTagInfo consists of all the subscription and
    /// publishing information
    /// </summary>
    /// 
-   public class PubSubState
-   {
+   public class PubSubState {
 
       public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent);
       public event PubSubStateSingleTagChangeDelegate onSingleTagChange;
@@ -33,8 +31,7 @@ namespace BCG
       private Dictionary<string, SingleTagData> myTagToData = new Dictionary<string, SingleTagData>();
       private Dictionary<string, SingleTagPubSubMap> myTagToPubsubMap = new Dictionary<string, SingleTagPubSubMap>();
 
-      public PubSubState()
-      {
+      public PubSubState() {
          SystemSubscriptionMapPublisher.init(this);
          SystemStatsPublisher.init(this);
          SystemHeartbeatPublisher.init();
@@ -49,13 +46,10 @@ namespace BCG
       /// </summary>
       /// <param name="sub"></param>
       /// <param name="md"></param>
-      public void sub(SubMetaData sub)
-      {
+      public void sub(SubMetaData sub) {
          ///tag based key ds update
-         foreach (string tag in sub.tags)
-         {
-            if (!myTagToPubsubMap.ContainsKey(tag))
-            {
+         foreach (string tag in sub.tags) {
+            if (!myTagToPubsubMap.ContainsKey(tag)) {
                myTagToPubsubMap.Add(tag, new SingleTagPubSubMap(tag));
             }
             myTagToPubsubMap[tag].onSub(sub);
@@ -67,20 +61,15 @@ namespace BCG
             onTagMapChanged(sub.tags);
       }
 
-      public void unsub(UnSubMetaData unsub)
-      {
-         foreach (string tag in unsub.tags)
-         {
-            if (myTagToPubsubMap.ContainsKey(tag))
-            {
+      public void unsub(UnSubMetaData unsub) {
+         foreach (string tag in unsub.tags) {
+            if (myTagToPubsubMap.ContainsKey(tag)) {
                ///maybe check here, that if there are no subscribers
                ///and publishers for the tag then remove it.
                myTagToPubsubMap[tag].onUnsub(unsub);
                if (onSingleTagChange != null)
                   onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
-            }
-            else
-            {
+            } else {
                string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
                   " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
                //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
@@ -90,11 +79,9 @@ namespace BCG
             onTagMapChanged(unsub.tags);
       }
 
-      public void removeConnection(string connection)
-      {
+      public void removeConnection(string connection) {
          List<string> tagsEffected = new List<string>();
-         foreach (KeyValuePair<string, SingleTagPubSubMap> p in myTagToPubsubMap)
-         {
+         foreach (KeyValuePair<string, SingleTagPubSubMap> p in myTagToPubsubMap) {
             ///maybe check here, that if there are no subscribers
             ///and publishers for the tag then remove it.
             bool wasPublisher = false;
@@ -103,8 +90,7 @@ namespace BCG
             if (p.Value.onConnectionRemoveIfExists(connection, out wasPublisher, out wasSubscriber))
                tagsEffected.Add(tag);
 
-            if (onSingleTagChange != null)
-            {
+            if (onSingleTagChange != null) {
                //we don't know the app or key qualifier or anything for these connection. that is unfortunate. 
                if (wasPublisher)
                   onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, ""));
@@ -128,67 +114,60 @@ namespace BCG
       /// </summary>
       /// <param name="tag"></param>
       /// <param name="msg"></param>
-      public void updatePubData(PubMetaData pub, Stream msg)
-      {
+      public void updatePubData(PubMetaData pub, Stream msg) {
          int sz = (int)(msg.Length - msg.Position);
+         bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
          ///this is the only place where SingleTagData should be created
-         if (!myTagToData.ContainsKey(pub.tag))
-         {
+         if (!myTagToData.ContainsKey(pub.tag)) {
             ///first time data is published for this tag
             myTagToData.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
          }
          myTagToData[pub.tag].pubUpdate(msg);
 
-         if (!myTagToPubsubMap.ContainsKey(pub.tag))
-         {
+         if (!myTagToPubsubMap.ContainsKey(pub.tag)) {
             myTagToPubsubMap.Add(pub.tag, new SingleTagPubSubMap(pub.tag));
          }
          bool pubChanged = false;
          myTagToPubsubMap[pub.tag].onPub(pub, out pubChanged);
-         if (pubChanged)
-         { 
+         if (pubChanged) {
             if (onTagMapChanged != null)
                onTagMapChanged(new string[] { pub.tag });
 
             if (onSingleTagChange != null)
                onSingleTagChange(
-                  new PubSubSingleTagStateEvent(pub.tag, 
-                     PubSubSingleTagStateEvent.EventType.publish, 
+                  new PubSubSingleTagStateEvent(pub.tag,
+                     PubSubSingleTagStateEvent.EventType.publish,
                      pub.connection, pub.publishingApp));
          }
          if (onPublishStats != null)
             onPublishStats(pub.tag, sz);
-         
+
       }
 
       #endregion
 
-      public ICollection<string> getSubscriberConnectionsForTag(string tag)
-      {
+      public ICollection<string> getSubscriberConnectionsForTag(string tag) {
          if (myTagToPubsubMap.ContainsKey(tag))
             return myTagToPubsubMap[tag].subscriberConnections;
          else
             return new string[] { };
       }
 
-      public ICollection<string> getSubscribingAppsForTag(string tag)
-      {
+      public ICollection<string> getSubscribingAppsForTag(string tag) {
          if (myTagToPubsubMap.ContainsKey(tag))
             return myTagToPubsubMap[tag].subscripingApps;
          else
             return new string[] { };
       }
 
-      public byte[] getContent(string tag)
-      {
+      public byte[] getContent(string tag) {
          if (!myTagToData.ContainsKey(tag))
             return null;
          else
             return myTagToData[tag].getCurrentContent();
       }
 
-      public string getTagType(string tag)
-      {
+      public string getTagType(string tag) {
          if (!myTagToData.ContainsKey(tag))
             return null;
          else
@@ -207,29 +186,23 @@ namespace BCG
       /// <param name="deadPublishers"></param>
       public void getAllConnections(string tag, out Dictionary<string, string> subscribers,
                                                 out Dictionary<string, string> activePublishers,
-                                                out ICollection<Pair<string, string>> deadPublishers)
-      {
-         if (myTagToPubsubMap.ContainsKey(tag))
-         {
+                                                out ICollection<Pair<string, string>> deadPublishers) {
+         if (myTagToPubsubMap.ContainsKey(tag)) {
             SingleTagPubSubMap map = myTagToPubsubMap[tag];
             subscribers = map.subscribers;
             activePublishers = map.activePublishers;
             deadPublishers = map.deadPublishers;
-         }
-         else
-         {
+         } else {
             subscribers = activePublishers = new Dictionary<string, string>();
             deadPublishers = new List<Pair<string, string>>();
          }
       }
 
-      public ICollection<string> allTags
-      {
+      public ICollection<string> allTags {
          get { return myTagToPubsubMap.Keys; }
       }
 
-      public ICollection<string> tagsWithData
-      {
+      public ICollection<string> tagsWithData {
          get { return myTagToData.Keys; }
       }
 
@@ -239,15 +212,11 @@ namespace BCG
       /// </summary>
       /// <param name="tags"></param>
       /// <returns></returns>
-      public Dictionary<string, List<string>> getSubscriberConnectionsForTags(IEnumerable<string> tags)
-      {
+      public Dictionary<string, List<string>> getSubscriberConnectionsForTags(IEnumerable<string> tags) {
          Dictionary<string, List<string>> retVal = new Dictionary<string, List<string>>();
-         foreach (string tag in tags)
-         {
-            if (myTagToPubsubMap.ContainsKey(tag))
-            {
-               foreach (string sub in myTagToPubsubMap[tag].subscriberConnections)
-               {
+         foreach (string tag in tags) {
+            if (myTagToPubsubMap.ContainsKey(tag)) {
+               foreach (string sub in myTagToPubsubMap[tag].subscriberConnections) {
                   if (!retVal.ContainsKey(sub))
                      retVal.Add(sub, new List<string>());
                   retVal[sub].Add(tag);

src/DataStreamSuite/PubSub/Server/SingleTagData.cs

@@ -7,21 +7,18 @@ using System.Collections.Generic;
 using BCG.Utilities;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SingleTagData consists of (as the name implies)
    /// the data published for a tag
    /// </summary>
-   internal class SingleTagData
-   {
+   internal class SingleTagData {
       private string myTagType = null;
       private string myTag;
       private byte[] myLastPublishedMsg;
       private IPubSubServerTagHandler myHandler;
 
-      public SingleTagData(string tag, string tagType)
-      {
+      public SingleTagData(string tag, string tagType) {
          myTag = tag;
          myTagType = tagType;
          if (myTagType == SystemSubscriptionMapPublisher.tagType)
@@ -29,6 +26,7 @@ namespace BCG
          else if (myTagType == SystemStatsPublisher.tagType)
             myHandler = SystemStatsPublisher.getInstance();
          else
+            // this is where it gets the stream handler
             myHandler = TagType2HandlerMap.getHandler(myTagType, true);
          if (myHandler != null)
             myHandler.register(new InternalPublishDelegate(myHandleHandlerCallbacks));
@@ -38,16 +36,13 @@ namespace BCG
 
       public string tagType { get { return myTagType; } }
 
-      internal void pubUpdate(Stream stream)
-      {
-         if (myHandler != null)
-         {
-            try
-            {
+      internal void pubUpdate(Stream stream) {
+         string handler_str = myHandler != null ? myHandler.GetType().ToString() : "NULL";
+         bts.Log.Gen.info($"SingleTagData.pubUpdate tag: {myTag} handler: {handler_str} stream: {stream.Position} / {stream.Length}");
+         if (myHandler != null) {
+            try {
                myHandler.update(myTag, new Stream[] { stream });
-            }
-            catch (Exception e)
-            {
+            } catch (Exception e) {
                // BlueEventLog.logEvent(e);
                string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + myTagType +
                                  " tag = " + myTag + " length = " + stream.Length + "\n" +
@@ -58,30 +53,22 @@ namespace BCG
                else
                   throw new ApplicationException(errMsg, e);
             }
-         }
-         else
-         {
+         } else {
             myLastPublishedMsg = new byte[stream.Length - stream.Position];
             stream.Read(myLastPublishedMsg, 0, myLastPublishedMsg.Length);
          }
       }
 
-      public byte[] getCurrentContent()
-      {
-         if (myHandler != null)
-         {
+      public byte[] getCurrentContent() {
+         if (myHandler != null) {
             return myHandler.getContent(myTag);
-         }
-         else
-         {
+         } else {
             return myLastPublishedMsg;
          }
       }
 
-      private void myHandleHandlerCallbacks(ICollection<Triplet<string, string, byte[]>> cbItems)
-      {
-         foreach (Triplet<string, string, byte[]> p in cbItems)
-         {
+      private void myHandleHandlerCallbacks(ICollection<Triplet<string, string, byte[]>> cbItems) {
+         foreach (Triplet<string, string, byte[]> p in cbItems) {
             string tagName = p.first, tagType = p.second;
             byte[] data = p.third;
             PubMetaData pub = new PubMetaData(tagType, tagName, data.Length, "local", ClientInfo.thisClient.processString);

src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs

@@ -7,8 +7,7 @@ using BCG.Utilities.General;
 using mbts;
 using mbts.spark;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Deals with the subscriptions (and getting data) from one
    /// pubsub server. Once again, theres no sharing of conenctions
@@ -17,15 +16,13 @@ namespace BCG
    /// its concerned with (passed as a conctructor arg).
    /// </summary>
 
-   internal interface ISingleServerSubscriberCore : IDisposable
-   {
+   internal interface ISingleServerSubscriberCore : IDisposable {
       IList<PubSubUpdate> getUpdates(int timeout);
       event PubSubConnectionChange onPubSubConnectionChange;
       void subscribe(params string[] tags);
       void unsubscribe(params string[] tags);
    }
-   internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore
-   {
+   internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
       private HighPerformanceTimer myTimer = new HighPerformanceTimer();
       private PubSubServerConnectionFactory myConnectionFactory;
       private IPubSubServerConnection myConnection;
@@ -33,15 +30,13 @@ namespace BCG
 
       public event PubSubConnectionChange onPubSubConnectionChange;
 
-      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory)
-      {
+      public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
          myConnectionFactory = connectionFactory;
          myConnection = myConnectionFactory.getConnection(serverURL);
          myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
       }
 
-      public IList<PubSubUpdate> getUpdates()
-      {
+      public IList<PubSubUpdate> getUpdates() {
          return getUpdates(1);
       }
 
@@ -51,54 +46,41 @@ namespace BCG
       /// </summary>
       /// <param name="timeout"></param>
       /// <returns></returns>
-      public IList<PubSubUpdate> getUpdates(int timeout)
-      {
+      public IList<PubSubUpdate> getUpdates(int timeout) {
          myConnection.doWork();
          List<PubSubUpdate> retVal = new List<PubSubUpdate>();
          IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
-         foreach (byte[] data in dataItems)
-         {
+         foreach (byte[] data in dataItems) {
             processMessage(data, retVal);
          }
          return retVal;
-         
+
       }
 
-      public List<string> getSubscribedTags()
-      {
+      public List<string> getSubscribedTags() {
          return myTags;
       }
 
-      public void subscribe(params string[] tags)
-      {
-         foreach (string tag in tags)
-         {
-            if (myTags.Contains(tag))
-            {
+      public void subscribe(params string[] tags) {
+         foreach (string tag in tags) {
+            if (myTags.Contains(tag)) {
                string errMsg = String.Format("tag {0} was already subscribed from server {1}",
                   tag, myConnection.serverURL);
                // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
-            }
-            else
-            {
+            } else {
                myTags.Add(tag);
             }
          }
          mySubscribe(tags);
       }
 
-      public void unsubscribe(params string[] tags)
-      {
+      public void unsubscribe(params string[] tags) {
          List<string> unsubTags = new List<string>();
-         foreach (string tag in tags)
-         {
-            if (myTags.Contains(tag))
-            {
+         foreach (string tag in tags) {
+            if (myTags.Contains(tag)) {
                myTags.Remove(tag);
                unsubTags.Add(tag);
-            }
-            else
-            {
+            } else {
                string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
                   "it was never subscribed", tag, myConnection.serverURL);
                // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
@@ -108,13 +90,13 @@ namespace BCG
          Stream str = new MemoryStream();
          new MetaData(unsub).save(str);
          str.Position = 0;
+         // this is where data is actually sent out
          myConnection.send(str);
       }
 
       #region IDisposable Members
 
-      public void Dispose()
-      {
+      public void Dispose() {
          ///remove the subscriber
          //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
          myConnection.Dispose();
@@ -126,8 +108,7 @@ namespace BCG
 
       #endregion
 
-      private void mySubscribe(params string[] tags)
-      {
+      private void mySubscribe(params string[] tags) {
          SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
          Stream str = new MemoryStream();
          new MetaData(sub).save(str);
@@ -135,20 +116,16 @@ namespace BCG
          myConnection.send(str);
       }
 
-      private void myHandleServerReboot(string serverURL, bool connection)
-      {
-         if (connection)
-         {
+      private void myHandleServerReboot(string serverURL, bool connection) {
+         if (connection) {
             //ie connection established again, so has to
             //resubscribe to all the tags
             // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
             //    PS3LogLevels.disconnection);
             mySubscribe(myTags.ToArray());
 
-            
-         }
-         else
-         {
+
+         } else {
             // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
             //    PS3LogLevels.disconnection);
          }
@@ -158,18 +135,17 @@ namespace BCG
             onPubSubConnectionChange(connection, serverURL);
       }
 
-      internal static void processMessage(byte[] data, List<PubSubUpdate> updates)
-      {
-         if (data != null && data.Length > 0)
-         {
+      internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
+         bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
+         if (data != null && data.Length > 0) {
             MemoryStream str = new MemoryStream(data);
             MetaData md = new MetaData(str);
+            bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage meta_data: {md}");
             MemoryStream bodyStream = str;
             long bodyPosition = str.Position;
-            object op = Activator.CreateInstance (Type.GetType("BCG." + md["op-code"]));
+            object op = Activator.CreateInstance(Type.GetType("BCG." + md["op-code"]));
             md.copyTo(op);
-            if (op is PubMetaData)
-            {
+            if (op is PubMetaData) {
                PubMetaData p = op as PubMetaData;
                //BCG.BlueTrace.WriteBlueCommReceiveW(p.tag, p.publishTimeTicks);
 
@@ -177,41 +153,29 @@ namespace BCG
                p.warnIfLatencyExceedsThreshold();
                //find out if we need to decompress this data
                Stream realBody = bodyStream;
-               if (p.isCompressed)
-               {
+               if (p.isCompressed) {
                   long compressedSize = bodyStream.Length - bodyStream.Position;
-                  if (compressedSize == 0)
-                  {
+                  if (compressedSize == 0) {
                      string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
                      Console.WriteLine(errorMsg);
                      //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
                      return;
-
-
                   }
-
                   realBody = StreamDecompressor.decompressStream(bodyStream);
                   realBody.Position = bodyPosition;
                }
-
-
-
                updates.Add(new PubSubUpdate(p.tag, p.type, realBody));
-            }
-            else if (op is TransactionMetaData)
-            {
+            } else if (op is TransactionMetaData) {
                TransactionMetaData tr = op as TransactionMetaData;
-               for (int i = 0; i < tr.nMessages; i++)
-               {
+               for (int i = 0; i < tr.nMessages; i++) {
                   PubMetaData pub = new PubMetaData();
                   new MetaData(bodyStream).copyTo(pub);
                   byte[] trConstituent = new byte[pub.size];
                   bodyStream.Read(trConstituent, 0, trConstituent.Length);
                   updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
                }
-            }
-            else{}
-               //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
+            } else { }
+            //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
          }
       }
    }

src/DataStreamSuite/PubSub/StaticPublisherCore.cs

@@ -15,91 +15,79 @@ using mbts;
  *       This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced. 
  *       
  * */
-namespace BCG
-{
-   public class CrushedTagManager
-   {
-      private static CrushedTagManager mySelf;
-      private Dictionary<string, bool> myTags = new Dictionary<string, bool>();
-      public static CrushedTagManager instance
-      {
-         get
-         {
-            if (mySelf == null)
-               mySelf = new CrushedTagManager();
-            return mySelf;
+namespace BCG {
+   // Provides a function to check if a tag is crushabel or not
+   public class CrushedTagManager {
+      private static CrushedTagManager _self;
+      private Dictionary<string, bool> _tag_map = new Dictionary<string, bool>();
+      public static CrushedTagManager instance {
+         get {
+            if (_self == null)
+               _self = new CrushedTagManager();
+            return _self;
          }
       }
-      public bool isTagCrushable(string tag)
-      {
+      public bool is_tag_crushable(string tag) {
          tag = tag.ToLower();
-         lock (typeof(CrushedTagManager))
-         {
-            if (!myTags.ContainsKey(tag))
-            {
+         bool rv = false;
+         lock (typeof(CrushedTagManager)) {
+            if (!_tag_map.TryGetValue(tag, out rv)) {
                foreach (string r in PubSub3Config.instance.PS3_CrushableTags)
-                  if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag))
-                  {
-                     myTags.Add(tag, true);
+                  if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) {
+                     _tag_map.Add(tag, true);
                      break;
                   }
-               if (!myTags.ContainsKey(tag))
-                  myTags.Add(tag, false);
+               _tag_map.Add(tag, false);
             }
          }
-         return myTags[tag];
+         return rv;
       }
    }
    /// <summary>
    /// All the data for a single tag, thats published within
    /// a transaction
    /// </summary>
-   public class TagWaitingInTransaction
-   {
-      private bool myIsCrushable;
-      private List<byte[]> myData = new List<byte[]>();
-      private int myTotalSize = 0;
-      private string myTag, myTagType;
-      public TagWaitingInTransaction(string theTagType, string theTag)
-      {
-         myTag = theTag;
-         myTagType = theTagType;
-         myIsCrushable = CrushedTagManager.instance.isTagCrushable(myTag);
+   public class TagWaitingInTransaction {
+      private bool _is_crushable;
+      private List<byte[]> _buffered_data = new List<byte[]>();
+      // this is the size of all the buffered data
+      private int _total_size = 0;
+      private string _tag, _tag_type;
+      public TagWaitingInTransaction(string theTagType, string theTag) {
+         _tag = theTag;
+         _tag_type = theTagType;
+         _is_crushable = CrushedTagManager.instance.is_tag_crushable(_tag);
       }
 
-      public void addPub(Stream pubStream)
-      {
-         if (myIsCrushable)
-         {
-            byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)];
-            pubStream.Read(newData, 0, newData.Length);
-            myData.Clear();
-            myData.Add(newData);
-            myTotalSize = newData.Length;
-         }
-         else
-         {
-            byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)];
-            pubStream.Read(newData, 0, newData.Length);
-            myData.Add(newData);
-            myTotalSize += newData.Length;
+      public void add_pub(Stream pub_stream) {
+         if (_is_crushable) {
+            // extract bytes, clear _buffered_data and queue to _buffered_data
+            byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
+            pub_stream.Read(new_data, 0, new_data.Length);
+            _buffered_data.Clear();
+            _buffered_data.Add(new_data);
+            _total_size = new_data.Length;
+         } else {
+            // extract btes and queue to _buffered_data
+            byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
+            pub_stream.Read(new_data, 0, new_data.Length);
+            _buffered_data.Add(new_data);
+            _total_size += new_data.Length;
          }
       }
 
-      public void writeTo(Stream s, string connection, string publishingApp)
-      {
-         new MetaData(new PubMetaData(myTagType, myTag, myTotalSize, connection, publishingApp)).save(s);
-         foreach (byte[] data in myData)
+      public void write_to(Stream s, string connection, string publishing_app) {
+         new MetaData(new PubMetaData(_tag_type, _tag, _total_size, connection, publishing_app)).save(s);
+         foreach (byte[] data in _buffered_data)
             s.Write(data, 0, data.Length);
       }
 
-      public string tag { get { return myTag; } }
-      public string tagType { get { return myTagType; } }
-      public List<byte[]> data { get { return myData; } }
+      public string tag { get { return _tag; } }
+      public string tagType { get { return _tag_type; } }
+      public List<byte[]> data { get { return _buffered_data; } }
    }
 
-   public enum PublisherCoreMessageType
-   {
+   public enum PublisherCoreMessageType {
       transactionBegin,
       transactionCommit,
       transactionAbort,
@@ -107,37 +95,35 @@ namespace BCG
       clear
    }
 
-   public class PublisherCoreMessage
-   {
+   public class PublisherCoreMessage {
       public PublisherCoreMessageType msgType;
       public string tagType, tag;
       public Stream data;
 
-      public PublisherCoreMessage(string tagToClear)
-      {
+      public PublisherCoreMessage(string tagToClear) {
          tag = tagToClear;
          msgType = PublisherCoreMessageType.clear;
 
       }
 
-      public PublisherCoreMessage(PublisherCoreMessageType theMsgType)
-      {
+      public PublisherCoreMessage(PublisherCoreMessageType theMsgType) {
          msgType = theMsgType;
       }
 
-      public PublisherCoreMessage(string theTagType, string theTag, Stream theData)
-      {
+      public PublisherCoreMessage(string theTagType, string theTag, Stream theData) {
          msgType = PublisherCoreMessageType.publish;
          tagType = theTagType;
          tag = theTag;
          data = theData;
       }
+      public override string ToString() {
+         return $"PublisherCoreMessage tag: {tag} data.Position: {data.Position} data.Length: {data.Length}";
+      }
    }
 
    // this class uses an internal queue + thread to serialize and process calls
    // made through its public interface (IPublisherCore)
-   public class StaticPublisherCore : IPublisherCore
-   {
+   public class StaticPublisherCore : IPublisherCore {
       // used to allow for different transports
       // to be used when communicating to a pubsubserver
       protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false);
@@ -172,57 +158,48 @@ namespace BCG
 
       public StaticPublisherCore()
          // TODOds
-         : this(new PubSubRouter())
-      { }
+         : this(new PubSubRouter()) { }
 
       public StaticPublisherCore(string serverURL)
-         : this(new PubSubRouter(new PubSubRoute(serverURL, ".*")))
-      { }
+         : this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) { }
 
-      public StaticPublisherCore(IPubSubRouter r)
-      {
+      public StaticPublisherCore(IPubSubRouter r) {
          myRouter = r;
          Thread pumpThread = new Thread(new ThreadStart(myMessagePump));
          pumpThread.IsBackground = true;
          pumpThread.Start();
       }
 
-      public void Dispose()
-      { }
+      public void Dispose() { }
       #endregion
 
       public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); }
 
       #region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
 
-      public void publish(string tag, Stream content)
-      { publish("", tag, content); }
+      public void publish(string tag, Stream content) { publish("", tag, content); }
 
-      public void publish(string tagType, string tag, Stream content)
-      {
-         myMainQueue.enqueue(new PublisherCoreMessage(tagType, tag, content));
+      public void publish(string tagType, string tag, Stream content) {
+         PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
+         bts.Log.Gen.info($"StaticPublisherCore.publish msg: {msg}");
+         myMainQueue.enqueue(msg);
       }
 
-      public void beginTransaction()
-      {
+      public void beginTransaction() {
          myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin));
       }
 
-      public void commitTransaction()
-      {
+      public void commitTransaction() {
          myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit));
       }
 
-      public void abortTransaction()
-      {
+      public void abortTransaction() {
          myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort));
       }
 
-      public bool inTransaction
-      { get { return myInTransaction; } }
+      public bool inTransaction { get { return myInTransaction; } }
 
-      public void clearState(string tag)
-      {
+      public void clearState(string tag) {
          myMainQueue.enqueue(new PublisherCoreMessage(tag));
       }
 
@@ -230,70 +207,51 @@ namespace BCG
 
       #region Message Pump Thread - this is where messages are dequeued and the action performed
 
-      private void myMessagePump()
-      {
-         while (true)
-         {
-            try
-            {
-               while (true)
-               {
+      private void myMessagePump() {
+         while (true) {
+            try {
+               while (true) {
                   object data = myMainQueue.dequeue(1000);
-                  if (data != null)
-                  {
+                  if (data != null) {
                      int qc = myMainQueue.getCount();
                      PublisherCoreMessage msg = data as PublisherCoreMessage;
-                     if (msg.msgType == PublisherCoreMessageType.transactionBegin)
-                     {
+                     bts.Log.Gen.info($"StaticPublisherCore.myMessagePump msg: {msg}");
+                     if (msg.msgType == PublisherCoreMessageType.transactionBegin) {
                         myBeginTransaction();
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.transactionCommit)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.transactionCommit) {
                         myCommitTransaction();
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.transactionAbort)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.transactionAbort) {
                         myAbortTransaction();
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.publish)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.publish) {
                         myPublish(msg.tagType, msg.tag, msg.data);
-                     }
-                     else if (msg.msgType == PublisherCoreMessageType.clear)
-                     {
+                     } else if (msg.msgType == PublisherCoreMessageType.clear) {
                         myClearState(msg.tag);
                      }
                   }
 
-                  foreach (IPubSubServerConnection conn in mySenders)
-                  {
+                  foreach (IPubSubServerConnection conn in mySenders) {
                      conn.doWork();
                   }
                }
-            }
-            catch (Exception ex)
-            {
-               // BlueEventLog.logEvent(ex);
+            } catch (Exception ex) {
+               bts.Log.Gen.error($"StaticPublisherCore.myMessagePump exception ex: {ex}");
                if (ex is OutOfMemoryException)
                   throw ex;
             }
          }
       }
 
-      private void myBeginTransaction()
-      {
+      private void myBeginTransaction() {
          // BlueEventLog.logAssert(!inTransaction);
          //BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information);
          myInTransaction = true;
       }
 
-      private void myCommitTransaction()
-      {
+      private void myCommitTransaction() {
          myEndTransaction(true);
       }
 
-      private void myAbortTransaction()
-      {
+      private void myAbortTransaction() {
          myEndTransaction(false);
       }
 
@@ -304,21 +262,17 @@ namespace BCG
       //keep it consistent with the way it was done earlier), this should
       //simply publish it non transactionally (easy way out), or 
       //transactionally with respect to each pubsub
-      protected virtual void myEndTransaction(bool success)
-      {
-         lock (this)
-         {
+      protected virtual void myEndTransaction(bool success) {
+         lock (this) {
             // BlueEventLog.logAssert(inTransaction);
             //BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information);
             myInTransaction = false;
-            if (success)
-            {
+            if (success) {
                // pubsub server -> tags with updates
                Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>();
 
                // group updates per pubsub server
-               foreach (string tag in myWaitingInTransactionMessages.Keys)
-               {
+               foreach (string tag in myWaitingInTransactionMessages.Keys) {
                   string thisURL = myRouter.getPubURLForTag(tag);
                   if (!routesToTags.ContainsKey(thisURL))
                      routesToTags[thisURL] = new List<string>();
@@ -327,17 +281,14 @@ namespace BCG
 
                // more than one server means we won't be able to honor the transactionality request
                // log a warning if that is the case.
-               if (routesToTags.Keys.Count > 1)
-               {
+               if (routesToTags.Keys.Count > 1) {
                   //this is going to be a long error message, but it's an importat one to be able to debug. 
                   StringBuilder errorMessageBuilder = new StringBuilder();
                   errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n");
                   errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n");
 
-                  foreach (string route in routesToTags.Keys)
-                  {
-                     foreach (string tag in routesToTags[route])
-                     {
+                  foreach (string route in routesToTags.Keys) {
+                     foreach (string tag in routesToTags[route]) {
                         errorMessageBuilder.Append("\tTag ");
                         errorMessageBuilder.Append(tag);
                         errorMessageBuilder.Append(" is routed to ");
@@ -351,16 +302,14 @@ namespace BCG
                }
 
                // iterate through each server with pending messages
-               foreach (KeyValuePair<string, List<string>> kvp in routesToTags)
-               {
+               foreach (KeyValuePair<string, List<string>> kvp in routesToTags) {
                   IPubSubServerConnection transactionSender = null;
                   string publishingApp = ClientInfo.thisClient.processString;
                   string connection = kvp.Key;
                   Stream transactionData = new MemoryStream();
 
                   // iterate through each of the pending tag updates for one server
-                  foreach (string tag in kvp.Value)
-                  {
+                  foreach (string tag in kvp.Value) {
                      ///it would be the same transaction sender that would 
                      ///return each time
                      transactionSender = myGetOrBuildSender(tag);
@@ -372,7 +321,7 @@ namespace BCG
                         // update internal state
                         myState.publish(waiting.tag, waiting.tagType, data);
                      // add a pubsub header and copy each message to the transactionData stream
-                     waiting.writeTo(transactionData, connection, publishingApp);
+                     waiting.write_to(transactionData, connection, publishingApp);
                   }
                   // now transactionData holds all the updates for all the tags 
                   // that are to be sent to the PubSub represented by transactionSender
@@ -384,36 +333,33 @@ namespace BCG
                   transactionData.Position = 0;
 
                   // send the entire thing to the specific pubsub server.
-                  transactionSender.send(new PubSubMessage(new MetaData(tMd), transactionData).toStream());
+                  // this is where the data is actually sent out !
+                  var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData);
+                  Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}");
+                  bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
+                  transactionSender.send(ps_msg);
                }
             }
             myWaitingInTransactionMessages.Clear();
          }
       }
 
-      protected virtual void myPublish(string tagType, string tag, Stream content)
-      {
-         if (myInTransaction)
-         {
+      protected virtual void myPublish(string tagType, string tag, Stream content) {
+         bts.Log.Gen.info($"StaticPublisherCore.myPublish tag: {tag} content: {content.Position} / {content.Length} myInTransaction: {myInTransaction}");
+         if (myInTransaction) {
             ///if its in transaction message is cached, and is sent
             ///only when theres a commit or abort
             if (!myWaitingInTransactionMessages.ContainsKey(tag))
                myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag));
-            myWaitingInTransactionMessages[tag].addPub(content);
-         }
-         else
-         {
-            //if (CrushedTagManager.instance.isTagCrushable(tag))
+            myWaitingInTransactionMessages[tag].add_pub(content);
+         } else {
             {
                myState.publish(tag, tagType, content);
-               if (!myCrushableTagsWaitingOnAck.ContainsKey(tag))
-               {
-                  // BlueEventLog.logEvent("Sending crushable tag " + tag, BlueEventLevel.Diagnosis);
+               if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
+                  bts.Log.Gen.info("Sending crushable tag: {tag}");
                   mySendData(tag, tagType, content);
-               }
-               else
-               {
-                  // BlueEventLog.logEvent("Could not send tag " + tag + " since its waiting on an ack", BlueEventLevel.Diagnosis);
+               } else {
+                  bts.Log.Gen.info("Could not send tag: {tag}  since its waiting on an ack");
                   myCrushableTagsWaitingOnAck[tag].second++;
                }
             }
@@ -428,64 +374,60 @@ namespace BCG
       }
 
       // add header, possibly compress the data, and sendto appropriate pubsub
-      private void mySendData(string tag, string tagType, Stream content)
-      {
+      private void mySendData(string tag, string tagType, Stream content) {
+         bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType}");
          long initPos = content.Position;
          // locate right pubsub for this tag
          IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
          Stream contentToPublish = content;
-         PubMetaData pubMd;
+         PubMetaData pub_md;
 
-         if (PubSub3Config.instance.isTagCompressed(tag))
-         {
+         if (PubSub3Config.instance.isTagCompressed(tag)) {
             contentToPublish = new MemoryStream();
             SharpZipCompressor Kompressor = new SharpZipCompressor();
             Kompressor.compress(content, contentToPublish);
             contentToPublish.Position = 0;
-            pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
-            if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag))
-            {
+            pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+            if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
                string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
                // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
             }
-         }
-         else
-            pubMd = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
+         } else
+            pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
 
          content.Position = initPos;
-         if (CrushedTagManager.instance.isTagCrushable(tag))
-         {
+         if (CrushedTagManager.instance.is_tag_crushable(tag)) {
             string refId = String.Empty;
             Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
-            serverConn.sendWithConfirmation((new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream() as MemoryStream).ToArray(), ref refId);
+            //  this is where the data is actually sent out (there are multiple sends)
+            PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
+            //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
+            bts.Log.Gen.info($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
+            bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
+            serverConn.send_with_confirmation(ps_msg, ref refId);
             myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
-         }
-         else
-         {
-            Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
-            serverConn.send(new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream());
+         } else {
+            //Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
+            // this is where the data is actually sent out
+            var ps_msg = new PubSubMessage(pub_md, contentToPublish);
+            bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
+            serverConn.send(ps_msg);
          }
       }
 
-      private void myClearState(string tag)
-      {
-         if (myTagToSender.ContainsKey(tag))
-         {
+      private void myClearState(string tag) {
+         if (myTagToSender.ContainsKey(tag)) {
             myTagToSender.Remove(tag);
          }
          myState.clear(tag);
-
       }
 
-      protected virtual IPubSubServerConnection myGetOrBuildSender(string tag)
-      {
-         if (!myTagToSender.ContainsKey(tag))
-         {
+      protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) {
+         if (!myTagToSender.ContainsKey(tag)) {
             string route = myRouter.getPubURLForTag(tag);
             IPubSubServerConnection sender = myConnectionFactory.getConnection(route);
             myTagToSender.Add(tag, sender);
-            if (!mySenders.Contains(sender))
-            {
+            if (!mySenders.Contains(sender)) {
                mySenders.Add(sender);
                sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
                sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
@@ -497,15 +439,11 @@ namespace BCG
 
       ///in case of the server coming up, it publishes all the stored state 
       ///for that server, back to it
-      protected void myHandleServerReboot(string serverURL, bool connection)
-      {
-         if (connection)
-         {
+      protected void myHandleServerReboot(string serverURL, bool connection) {
+         if (connection) {
             string thisPublisherInfo = ClientInfo.thisClient.processString;
-            foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender)
-            {
-               if (p.Value.serverURL == serverURL)
-               {
+            foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
+               if (p.Value.serverURL == serverURL) {
                   string tagType;
                   string tag = p.Key;
                   myCrushableTagsWaitingOnAck.Clear();
@@ -547,16 +485,12 @@ namespace BCG
       /// of waiting for ack
       /// </summary>
       private Dictionary<string, Pair<string, int>> myCrushableTagsWaitingOnAck = new Dictionary<string, Pair<string, int>>();
-      private void myOnSendConfirm(string messageId, bool confirmed)
-      {
-         foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys))
-         {
+      private void myOnSendConfirm(string messageId, bool confirmed) {
+         foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys)) {
             Pair<string, int> tagData = myCrushableTagsWaitingOnAck[tag];
-            if (messageId == tagData.first)
-            {
+            if (messageId == tagData.first) {
                myCrushableTagsWaitingOnAck.Remove(tag);
-               if (tagData.second > 0)
-               {
+               if (tagData.second > 0) {
                   // BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis);
                   /// this data was waiting on an
                   /// ack, need to resend it

src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs

@@ -0,0 +1,132 @@
+using System;
+using System.Linq.Expressions;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading.Tasks;
+using System.IO;
+
+using Spark.Core.Reactive;
+using System.Security.Cryptography.X509Certificates;
+using BCG;
+
+namespace BCG
+{
+
+   // This uses AsyncTCPConnection2's stateful async functions
+   public class SimpleTCPClientAsync : ITCPConnectionHandler, ITCPMessageHandler
+   {
+      AsyncTCPConnection2 _conn = null;
+      int _m_msgs_received = 0;
+      public static async Task start(string ip, int port)
+      {
+         SimpleTCPClientAsync c = new SimpleTCPClientAsync(ip, port);
+         c.connect();
+      }
+
+      public void connect()
+      {
+         Task.Run(async () => await TCPSocketUtils.connect_to_server_async("127.0.0.1", 12345, this, TimeSpan.FromSeconds(2)));
+      }
+
+      public void onConnection(ITCPConnection connection)
+      {
+         Console.WriteLine("SimpleTCPClientAsync.onConnection connected to server, starting task to send data in a loop");
+         connection.register(this);
+         // start sending data
+         Task.Run(async () =>
+         {
+            byte[] buff = new byte[1024];
+            for (int i = 0; i < buff.Length; i++)
+            {
+               buff[i] = (byte)i;
+            }
+            _conn = connection as AsyncTCPConnection2;
+            while (!_conn.isDisposed)
+            {
+               await _conn.send_async(buff);
+               await Task.Delay(100);
+            }
+         });
+      }
+
+      public void onData(RemoteMessage msg)
+      {
+         _m_msgs_received++;
+         Console.WriteLine($"SimpleTCPClientAsync.onData n_msgs: {_m_msgs_received} msg: {msg}");
+         if (msg.messageType == RemoteMessageType.Disconnection)
+         {
+            connect();
+         }
+      }
+
+      public bool isDisposed { get { return false; } }
+
+      public void Dispose() { }
+
+      public SimpleTCPClientAsync(string ip, int port) { }
+
+   }
+   public class SimpleTCPClient : ITCPConnectionHandler, ITCPMessageHandler
+   {
+      private int _n_msgs_received = 0;
+      AsyncTCPConnection2 _conn = null;
+
+      public static void start(string ip, int port)
+      {
+         SimpleTCPClient c = new SimpleTCPClient(ip, port);
+         c.connect();
+      }
+
+      public void connect()
+      {
+         Console.WriteLine("SimpleTCPClient.onConnection connect() called");
+         ClientToServerManagedTCPConnector.connect_to(new BlutoSubAddress("bluto://127.0.0.1:12345"), this, TimeSpan.FromSeconds(1));
+      }
+
+      public void onConnection(ITCPConnection connection)
+      {
+         _conn = connection as AsyncTCPConnection2;
+         Console.WriteLine("SimpleTCPClient.onConnection connected to server, starting task to send data in a loop");
+         _conn.register(this);
+         byte[] buff = new byte[1024];
+         for (int i = 0; i < buff.Length; i++)
+         {
+            buff[i] = (byte)i;
+         }
+
+
+         while (!_conn.isDisposed)
+         {
+            _conn.send(buff);
+            System.Threading.Thread.Sleep(1000);
+         }
+
+         // Task.Run(async () =>
+         // {
+         //    while (!_conn.isDisposed)
+         //    {
+         //       await _conn.send_async(buff);
+         //       await Task.Delay(100);
+         //    }
+         // });
+      }
+
+      public void onData(RemoteMessage msg)
+      {
+         _n_msgs_received++;
+         Console.WriteLine($"SimpleTCPClient.onData messages received: {_n_msgs_received} msg: {msg} messages received: {_n_msgs_received}");
+         if (msg.messageType == RemoteMessageType.Disconnection)
+         {
+            connect();
+         }
+      }
+
+      public bool isDisposed { get { return false; } }
+
+      public void Dispose() { }
+
+      public SimpleTCPClient(string ip, int port) { }
+
+   }
+}

src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs

@@ -0,0 +1,123 @@
+using BCG;
+
+using Google.Protobuf;
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BCG;
+using System.Xml.Serialization;
+using System.Reflection;
+using System.Data;
+using System.Collections.Generic;
+
+namespace BCG
+{
+
+   public class SimpleTCPServerAsync : ITCPConnectionHandler, ITCPMessageHandler
+   {
+      int _n_msgs_received = 0;
+      SingleServerToClientManagedTCPConnectionAcceptor _acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+      IDictionary<RemoteConnectionInfo, ITCPConnection> _conn_map = new Dictionary<RemoteConnectionInfo, ITCPConnection>();
+
+      public static async Task start()
+      {
+         Console.WriteLine("SimpleTCPServer.start accepting socket connections");
+
+         SimpleTCPServerAsync s = new SimpleTCPServerAsync();
+         //await AsyncTCPConnection2.accept_connections("127.0.0.1", 12345, s, TimeSpan.FromMilliseconds(100));
+         s.accept();
+      }
+
+      // public static async Task init()
+      // {
+      // }
+
+      public void accept()
+      {
+         _acceptor.accept_on_port(this, "127.0.0.1", 12345, TimeSpan.Zero);
+      }
+
+      public void onConnection(ITCPConnection connection)
+      {
+         AsyncTCPConnection2 new_conn = connection as AsyncTCPConnection2;
+         RemoteConnectionInfo rci = new_conn.get_peer_address();
+         _conn_map.Add(rci, new_conn);
+         Console.WriteLine($"SimpleTCPServerAsync.onConnection new client connected conn: {rci}");
+         new_conn.register(this);
+      }
+
+
+      public void onData(RemoteMessage msg)
+      {
+         Console.WriteLine($"SimpleTCPServerAsync.onData msg: {msg}");
+         RemoteConnectionInfo rci = msg.remoteConnectionInfo;
+         ITCPConnection this_conn;
+         bool found = _conn_map.TryGetValue(rci, out this_conn);
+         if (!found)
+         {
+            Console.WriteLine("Could not find connection !!!!!!");
+            return;
+         }
+         _n_msgs_received++;
+         Console.WriteLine($"SimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} echoing it back");
+         if (msg.messageType == RemoteMessageType.Data)
+         {
+
+            this_conn.send(msg.message);
+            // Task.Run(async () => {
+            //    Console.WriteLine($"SimpleTCPServerAsync.onData sending msg: {msg} to client");
+            //    await _conn.send_async(msg.message);
+            // });
+         }
+      }
+
+      public bool isDisposed { get { return false; } }
+
+      public void Dispose() { }
+   }
+
+   //    public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler {
+   //       private AsyncTCPConnection2 _conn = null;
+   //       private int _n_msgs_received = 0;
+   //       SingleServerToClientManagedTCPConnectionAcceptor _acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
+
+   //       public static async Task start() {
+   //          Console.WriteLine("SimpleTCPServer.start accepting socket connections");
+   //          SimpleTCPServer s = new SimpleTCPServer();
+   //          s.accept();
+   //          //await AsyncTCPConnection2.accept_connections("127.0.0.1", 12345, s, TimeSpan.FromMilliseconds(100));
+   //       }
+
+   //       public void accept() {
+   //          _acceptor.accept_on_port(this, "127.0.0.1", 12345, TimeSpan.Zero);
+   //       }
+
+   //       public void onConnection(ITCPConnection connection) {
+   //          Console.WriteLine("SimpleTCPServerAsync.onConnection new client connected");
+   //          _conn = connection as AsyncTCPConnection2;
+   //          connection.register(this);
+   //       }
+
+   //       public async void onData(RemoteMessage msg) {
+   //          _n_msgs_received++;
+   //          Console.WriteLine($"SimpleTCPServerAsync.onData msg: {msg} , msgs received: {_n_msgs_received} echoing it back");
+   //          if (msg.messageType == RemoteMessageType.data && _conn != null) {
+   //             Task.Run(async () => {
+   //                //Console.WriteLine($"SimpleTCPServerAsync.onData sending msg: {msg} to client");
+   //                //await _conn.send_async(msg.message);
+   //                _conn.send(msg.message);
+   //             });
+   //          }
+   //       }
+
+   //       public bool isDisposed { get { return false; } }
+
+   //       public void Dispose() { }
+   //    }
+}

src/DataStreamSuite/PubSub/Testing/TestPublisher.cs

@@ -0,0 +1,52 @@
+using BCG;
+
+using bts;
+
+using System;
+using System.IO;
+using System.Runtime.Serialization;
+
+namespace BCG {
+   public class TestPubItem : IPubItem {
+      int i = 100;
+
+
+      public static string test_tag = "TestTag";
+      public string getTag() {
+         return test_tag;
+      }
+
+      public void to_stream(Stream str) {
+         BinaryWriter bw = new BinaryWriter(str);
+         //bw.Write(d);
+         bw.Write(i);
+      }
+   }
+   public class TestPublisher {
+      TestPubItem _tbi = new TestPubItem();
+      IPublisherCore pcore = new StaticPublisherCore();
+      bool _hit_key_to_send = false;
+      public TestPublisher(bool hit_key_to_send) {
+         _hit_key_to_send = hit_key_to_send;
+         bts.Log.Gen.info($"Started TestPubliser _hit_key_to_send: {_hit_key_to_send}");
+      }
+
+      public void keep_publishing(TimeSpan wait_per, int n_pubs = System.Int32.MaxValue) {
+         TestPubItem pi = new TestPubItem();
+         for (int i = 0; i < n_pubs; i++) {
+            if (_hit_key_to_send) {
+               Console.WriteLine("Hit key to send data");
+               Console.ReadLine();
+            }
+            MemoryStream ms = new MemoryStream();
+            pi.to_stream(ms);
+            ms.Position = 0;
+            pcore.beginTransaction();
+            pcore.publish(TestPubItem.test_tag, ms);
+            pcore.commitTransaction();
+            if (!_hit_key_to_send)
+               System.Threading.Thread.Sleep(wait_per);
+         }
+      }
+   }
+}

src/DataStreamSuite/PubSub/Testing/TestSubscriber.cs

@@ -0,0 +1,31 @@
+using BCG;
+
+using bts;
+
+using System;
+using System.IO;
+using System.Runtime.Serialization;
+
+namespace BCG {
+
+   public class TestSubscriber {
+      SubscriberCore _sub_core = new SubscriberCore();
+      public TestSubscriber() {
+         bts.Log.Gen.info("Started TestSubscriber");
+      }
+
+      public void keep_polling(TimeSpan wait_per, int n_polls = System.Int32.MaxValue) {
+         _sub_core.subscribe(TestPubItem.test_tag);
+         for (int i = 0; i < n_polls; i++) {
+            var list = _sub_core.getUpdates();
+            if (list.Count > 0) {
+               Console.WriteLine($"Received {list.Count} items");
+            }
+            foreach (var p in list) {
+               Console.WriteLine($"Received item: {p} iter: {i}");
+            }
+            System.Threading.Thread.Sleep(wait_per);
+         }
+      }
+   }
+}

src/DataStreamSuite/PubSubServer/Program.cs

@@ -1,22 +1,24 @@
 using System;
 using System.Threading;
+using System.Threading.Tasks;
 using System.Linq;
 
 //using BCG.Utilities;
 //using BCG.Utilities.General;
 
 
-namespace BCG
-{
-   class PubSub2Main
-   {
+namespace BCG {
+   class PubSub2Main {
 
       /// <summary>
       /// The main entry point for the application.
       /// </summary>
       [STAThread]
-      public static void Main(string[] args)
-      {
+      public static async Task Main(string[] args) {
+         bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
+         mbts.config.Configuration.initialize();
+         // SimpleTCPServerAsync.start();
+         // Task.Delay(1000000000).Wait();
          if (args.Length > 0 && args[0] == "daemon")
             args = args.Skip(1).ToArray();
 
@@ -26,14 +28,11 @@ namespace BCG
          long currentMemory = 0;
          int defaultWaitTime = 100;
          bool gced = false;
-         new Thread(() =>
-         {
-            while (true)
-            {
+         new Thread(() => {
+            while (true) {
                gced = false;
                currentMemory = GC.GetTotalMemory(false);
-               if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold)
-               {
+               if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold) {
                   string gcMsg = String.Format("GCing memory, mem before GC = {0}", currentMemory);
                   // BlueEventLog.logEvent(gcMsg, PS3LogLevels.info);
                   Console.WriteLine(gcMsg);

src/lib/config/protos/data_stream_config.proto

@@ -1,19 +0,0 @@
-syntax = "proto3";
-
-package Bts.DataStream;
-
-option csharp_namespace = "Bts.DataStream";
-
-message DataStreamConfig {
-  repeated string assemblies = 1;
-  bool use_static_parser = 2;
-  bool log_blob_packets = 3;
-  bool is_test_mode = 4;
-  bool profile_streams = 5;
-  double profile_threshold = 6;
-  bool use_new_filter = 7;
-  bool use_new_map = 8;
-  bool use_increments_for_multiple_ops = 9;
-  bool use_uncopied_new_rows_in_fire = 10;
-  bool use_modularized_data_stream_bridge = 11;
-}