Fixing AsyncTCPCOnnection_APM's double partial messages for messages > 100 MB

Good riddance. Improper 'PlatformMap' config was often the reason market
data mysteriously wouldn't show up in the client. Especially with new
exchanges. We should keep all md source/connection configuration in one
place (one proto message), to avoid any confusion.

.gitignore

@@ -47,8 +47,8 @@ trace_data/
 /src/.clang-format
 /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
 /src/server/sparkd/cmds/spartan/analysis/data
-*.tmp/*
-/.venv*/*
+.tmp/
+.venv*/
 .cache/
 /src/Spark.Admin.WebApp/tsconfig.*.g.json
 /src/Spark.Admin.WebApp/src/bin/

src/DataStreamSuite/DataStream/DataStream.csproj

@@ -15,6 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
     <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
     <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
   </ItemGroup>

src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs

@@ -1,32 +0,0 @@
-using BCG;
-
-namespace BCG
-{
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("add_to_cloud")]
-   public class AddToCloudStream : ViewStream
-   {
-      private readonly string _name, _stream_exp;
-      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
-      public AddToCloudStream(string name, params string[] lines)
-      : this(name, 0, lines)
-      {
-      }
-
-      public AddToCloudStream(string name, int persist, params string[] lines)
-      {
-         _name = name;
-         _stream_exp = getExpression(lines);
-         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
-         mySetRealSource(new TableStream("cloud_streams"));
-      }
-
-      static string getExpression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-   }
-}

src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs

@@ -9,23 +9,20 @@ using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
 using System.Collections.Generic;
 using BCG.Utilities.General;
+using bts;
 
-namespace BCG
-{
+namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class CloudStreamManager : IWorker
-   {
-      public class StreamPubUnit
-      {
+   public class RemoteQueryManager : IWorker {
+      public class StreamPubUnit {
          public string name;
          public string stream_expression;
          public ViewStream stream;
          public bool persist;
          public ViewPublisher view_publisher;
 
-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
-         {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
             name = name_;
             stream_expression = stream_expression_;
             stream = stream_;
@@ -33,17 +30,13 @@ namespace BCG
             view_publisher = view_publisher_;
          }
       }
-      static ViewStream build_stream(string stream_expression)
-      {
-         try
-         {
+      static ViewStream build_stream(string stream_expression) {
+         try {
             var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
             var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
             return rv;
 
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
             return null;
          }
@@ -52,34 +45,34 @@ namespace BCG
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
       private ViewStream _stream = new TableStream("cloud_streams");
       private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;
 
-      public CloudStreamManager()
-      {
+
+      public RemoteQueryManager() {
          _stream.onChange += _on_str;
-
+         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
       }
-      private void _on_str(object obj)
-      {
+      // add a RemoteQuery sub, additionally and subscribe to it
+      private void _on_str(object obj) {
          DataPacket dp = obj as DataPacket;
-         foreach (IBlueRow r in dp.newRows)
-         {
+         foreach (IBlueRow r in dp.newRows) {
             string name = r.get<string>("name");
             bool persist = r.get<bool>("persist");
             string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name))
-            {
-               _assign(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
             }
          }
       }
 
-      private void _assign(string name, string stream_expression, bool persist)
-      {
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
          bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
          ViewStream stream = build_stream(stream_expression);
          _expose_to_ws(name, stream);
-         if (persist)
-         {
+         if (persist) {
             stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
          }
          ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -87,13 +80,11 @@ namespace BCG
          _stream_pub_map.Add(name, u);
       }
 
-      private void _expose_to_ws(string name, ViewStream stream)
-      {
+      private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
 
-      public void doWork()
-      {
+      public void doWork() {
 
       }
    }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -0,0 +1,48 @@
+using BCG;
+
+using bts;
+
+using System;
+
+namespace BCG {
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
+   public class RemoteQueryStream : ViewStream {
+      private readonly string _name, _stream_exp;
+      private PublisherCore _publisher_core = new PublisherCore();
+      private ViewStream _source_str;
+
+      public RemoteQueryStream(params string[] lines) {
+         _stream_exp = get_expression(lines);
+         _name = Guid.NewGuid().ToString();
+         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
+         //_name.Replace("-", "_");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
+         byte[] rem_req_bytes = rem_req.serialize();
+         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
+      }
+
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
+
+      static string get_expression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+
+      private void _handle_data(object o) {
+         myFireEvent(o);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs

@@ -0,0 +1,31 @@
+namespace bts {
+   using System.IO;
+   using BCG;
+
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
+   public class RemoteRequest {
+      public readonly string stream_expression;
+      public readonly string published_table_name;
+      public RemoteRequest(string stream_expression_, string published_table_name_) {
+         stream_expression = stream_expression_;
+         published_table_name = published_table_name_;
+      }
+      public byte[] serialize() {
+         var str = new MemoryStream();
+         var bw = new BinaryWriter(str);
+         bw.write_string_cpp(stream_expression);
+         bw.write_string_cpp(published_table_name);
+         return str.ToArray();
+      }
+
+      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
+         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
+         string stream_expression_ = br.read_string_cpp();
+         string published_table_name_ = br.read_string_cpp();
+         return new RemoteRequest(stream_expression_, published_table_name_);
+
+      }
+   }
+}

src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs

@@ -1,59 +1,54 @@
-using BCG;
+// using BCG;
 
-using System.Linq;
+// using bts;
 
-namespace BCG
-{
-   public class StreamOffloadRequestor
-   {
-      public class OffloadOp
-      {
-         string name;
-         string stream_exp;
-      }
-      private IBlueTableSchema _schema;
-      private IBlueRowCollection _rows;
-      private PubTableBroker _broker = new PubTableBroker();
-      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-      public StreamOffloadRequestor()
-      {
-         _schema = new BlueTableSchema("cloud_streams",
-                                   new ColumnInfo("id", 0, typeof(int)),
-                                   new ColumnInfo("name", 1, typeof(string)),
-                                   new ColumnInfo("persist", 2, typeof(bool)),
-                                   new ColumnInfo("stream_exp", 3, typeof(string)));
-         _existing_offloads_vs.onChange += _on_str;
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         _rows = dp.newRows;
-         // this is needed so the broker knows the state of the table
-         _broker.createTable(_rows);
-      }
-      public void add_to_cloud(string name, string stream_exp, bool persist)
-      {
-         // check if it exists
-         if (_rows == null)
-         {
-            BlueTable table = new BlueTable(_schema);
-            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-            table.addRow(nr);
-            _broker.createTable(table);
-            // create table  
-         }
-         else
-         {
-            if (exists(name))
-               return;
-            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-            _broker.addRow(nr);
-         }
-      }
+// using System.Linq;
 
-      bool exists(string name)
-      {
-         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-      }
-   }
-}
+// namespace BCG {
+//    // used on the client
+//    public class StreamOffloadRequestor {
+//       public class OffloadOp {
+//          string name;
+//          string stream_exp;
+//       }
+//       private IBlueTableSchema _schema;
+//       private IBlueRowCollection _rows;
+//       private PubTableBroker _broker = new PubTableBroker();
+//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+//       public StreamOffloadRequestor() {
+//          _schema = new BlueTableSchema("cloud_streams",
+//                                    new ColumnInfo("id", 0, typeof(int)),
+//                                    new ColumnInfo("name", 1, typeof(string)),
+//                                    new ColumnInfo("persist", 2, typeof(bool)),
+//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
+//          _existing_offloads_vs.onChange += _on_str;
+//       }
+//       private void _on_str(object obj) {
+//          DataPacket dp = obj as DataPacket;
+//          _rows = dp.newRows;
+//          // this is needed so the broker knows the state of the table
+//          _broker.create_table(_rows);
+//       }
+//       public void add_to_cloud(string name, string stream_exp, bool persist) {
+//          // check if it exists
+//          if (_rows == null) {
+//             BlueTable table = new BlueTable(_schema);
+//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+//             table.addRow(nr);
+//             _broker.create_table(table);
+//             // create table  
+//          } else {
+//             if (exists(name))
+//                return;
+//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+//             // TODO: use make_batch_changes instead
+//             Log.assert(false, "Fix addRow");
+//             //_broker.addRow(nr);
+//          }
+//       }
+
+//       bool exists(string name) {
+//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs

@@ -1,17 +1,14 @@
 using System;
 using System.Collections;
 
-namespace BCG
-{
-   internal class AddColumnsRow : IBlueRow
-   {
+namespace BCG {
+   internal class AddColumnsRow : IBlueRow {
       private IBlueTableSchema mySchema;
       private IBlueRow myRow;
       //private object[] myCachedValues;
 
       #region Constructors
-      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
-      {
+      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
          mySchema = schema;
          myRow = row;
          //myCachedValues = new object[schema.columnNames.Length];
@@ -19,85 +16,78 @@ namespace BCG
       #endregion
 
       #region IBlueRow members
-      public IBlueTableSchema schema{get{return mySchema;}}
+      public IBlueTableSchema schema { get { return mySchema; } }
       #endregion
 
       #region ICollection members
-      public int Count {get{return mySchema.columnNames.Length;}}
+      public int Count { get { return mySchema.columnNames.Length; } }
 
-      public bool IsSynchronized {get{return false;}}
+      public bool IsSynchronized { get { return false; } }
 
-      public object SyncRoot {get{return this;}}
+      public object SyncRoot { get { return this; } }
 
-      public void CopyTo (Array array, int index)  
-      {
+      public void CopyTo(Array array, int index) {
          int i = index;
          foreach (object o in this as IEnumerable)
-            array.SetValue (o, i++);
+            array.SetValue(o, i++);
       }
       #endregion
 
       #region IDictionary members
-      public bool IsFixedSize {get{return true;}}
+      public bool IsFixedSize { get { return true; } }
 
-      public bool IsReadOnly {get{return true;}}
+      public bool IsReadOnly { get { return true; } }
 
       /// <summary>
       /// The key can either be the string corrsponding to the column name or
       /// the order of the column
       /// </summary>
-      public object this [object key]
-      {
-         get
-         {
+      public object this[object key] {
+         get {
             if (key is string)
                return this[(string)key];
             return this[(int)key];
          }
-         set {throw new NotSupportedException ("The IBlueRow is readonly");}
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
       }
 
-      public object this[int index]
-      {
-         get
-         {
+      public object this[int index] {
+         get {
             //if (myCachedValues[index] == null)
             //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
             //return myCachedValues[index];
-            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
          }
       }
 
-      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
+      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
 
-      public ICollection Keys{get{return mySchema.columnNames;}}
+      public ICollection Keys { get { return mySchema.columnNames; } }
 
-      public ICollection Values
-      {
-         get
-         {
+      public ICollection Values {
+         get {
             object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo (retVal, 0);
+            CopyTo(retVal, 0);
             return retVal;
          }
       }
 
-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
 
-      public bool Contains (object key){return mySchema[(string)key] != null;}
+      public bool Contains(object key) { return mySchema[(string)key] != null; }
 
       /// <summary>
       /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
       /// iterate through the values, typecast this into an IEnumerable
       /// </summary>
       /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
-      
-      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
 
-      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
       #endregion
 
    }

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs

@@ -1,11 +1,38 @@
-// using System;
-// using System.Data;
-// using System.Collections;
-// using BCG.Utilities;
-// using BCG.Utilities.General;
-// namespace BCG {
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using BCG.Utilities.General;
+namespace BCG {
 
-//     public class AddColumnsStream : MapStream {
-//         public AddColumnsStream(ViewStream vs, string columnDef) : base(vs, new ColumnAdder(columnDef)) { }
-//     }
-// }
+   public class AddColumnsStream : ViewStream {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
+         string[] toks = exp.Split(':');
+         string col_name = toks[0];
+         string col_value = toks[1];
+         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+      }
+
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
+
+      public AddColumnsStream(ViewStream vs, string col_def) {
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         mySetRealSource(rs);
+      }
+
+      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         return _out_schema;
+      }
+
+      private IBlueRow _map(IBlueRow r) {
+         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
+         rv.setValue(_new_col.Item1, _new_col.Item3);
+         return rv;
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,48 +1,24 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
+using BCG;
 
-using System.Runtime.Serialization.Formatters.Binary;
-
-namespace BCG
-{
+namespace bts {
    [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream, ISideEffectStream
-   {
-      public class Exception : System.Exception
-      {public Exception(string msg) : base(msg) { }}
+   public class LoaderStream : ViewStream {
 
-      public LoaderStream(string fileName, int secondsBetweenPolls)
-      {
-         mySetRealSource(
-            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-               ViewStreamDatabase.instance));
+      private string _filename;
+      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+
+
+      public LoaderStream(string filename) {
+         _filename = filename;
       }
 
-      public LoaderStream(string fileName)
-      {
-        if (fileName == null)
-           throw new LoaderStream.Exception("File name cannot be null");
-
-        for (int i = 0; i < 5; i++)
-           try
-           {
-              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-              {
-                  BinaryFormatter bf = new BinaryFormatter();
-#pragma warning disable SYSLIB0011
-                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
-#pragma warning restore SYSLIB0011
-                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
-              }
-              break;
-           }
-           catch (IOException)
-           {
-              System.Threading.Thread.Sleep(50);
-           }
-
+      protected override void myOnFirstSubscriber() {
+         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
+         StreamIOMessage msg = _deser.deserialize(ser_msg);
+         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
+         IBlueRowCollection rows = msg.create_message.rows;
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
+using bts;
 
 namespace BCG {
    // Writes all the udpates to a stream in a file.
@@ -19,7 +20,7 @@ namespace BCG {
       private ViewStream _source_stream;
       private string _token;
       private FileStream _out_fs;
-      private IBlueDataSerializer _serializer;
+      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
       private string _file_name = String.Empty;
       private long _rollover_size;
       private readonly string _directory_path;
@@ -89,13 +90,12 @@ namespace BCG {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
          bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         _serializer = _serializer ?? new UniversalStreamIOSerializer();
          if (_write_create) {
             // new files always starts with create
-            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
-            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
-
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
          BinaryWriter bw = new BinaryWriter(_out_fs);
          long utc_epoch = epoch_now();
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +1,68 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
 
-namespace BCG
-{
-   public class RecorderStream : ViewStream
-   {
-      private ViewStream mySource;
-      private DateTime myStartTime;
-      private FileStream myOutFile;
-      private string myFileName;
-      private IBlueDataSerializer mySerializer;
-            
-      public RecorderStream(ViewStream src, string recordFile)
-      {
-         mySource = src;
-         myFileName = recordFile;
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
 
-         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-      }
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
 
-      protected override void myOnFirstSubscriber()
-      {
-         myStartTime = DateTime.Now;         
-         mySource.onChange += new ChangeDelegate(mySource_onChange);
-         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-      }
-      
-      protected override void myOnLastSubscriber()
-      {
-         mySource.onChange -= mySource_onChange;
-         myOutFile.Close();
-      }
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
 
-      void mySource_onChange(object newValue)
-      {
-         DataPacket dp = (DataPacket)newValue;         
-         
-         BlueTable toWrite = new BlueTable(dp.schema);
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
 
-         if (mySerializer == null)
-         {
-            mySerializer = new UniversalSerializer(dp.schema);
-         }
-         foreach (IBlueRow r in dp.newRows)
-            toWrite.addRow(r);
-         
-         byte[] result;
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
 
-         if (myLastValue == null)
-            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-         else
-            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
 
-         BinaryWriter bw = new BinaryWriter(myOutFile);
-         //write down what time this happened
-         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-         bw.Write(result.Length);
-         bw.Write(result);         
-         myOutFile.Flush();
+//          BlueTable toWrite = new BlueTable(dp.schema);
 
-         myFireEvent(dp);
-      }
-   }
-}
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -1,64 +1,37 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BCG;
 
-// using System.Runtime.Serialization.Formatters.Binary;
+namespace bts {
+   // this saves the total state in the filr provided in the cons
+   [DataStreamUserRespresentation("saver")]
+   public class SaverStream : ViewStream {
+      private ViewStream _stream;
+      private string _filename;
+      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
 
-// namespace BCG
-// {
-//    [DataStreamUserRespresentation("saver")]
-//    public class SaverStream : ViewStream, ISideEffectStream
-//    {
-//       private ViewStream myStream;
-//       private string myFileName;
+      public SaverStream(ViewStream vs, string filename) {
+         _stream = vs;
+         _filename = filename;
+      }
 
-//       public class Exception : System.Exception
-//       {
-//          public Exception(string msg) : base(msg) { }
-//       }
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
 
-//       public SaverStream(ViewStream vs, string fileName)
-//       {
-//          if (fileName == null)
-//             throw new SaverStream.Exception("File name cannot be null");
-//          myStream = vs;
-//          myFileName = fileName;
+      void _handle_changes(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _save(dp);
+         myFireEvent(dp_obj);
+      }
 
-//       }
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
 
-//       protected override void myOnFirstSubscriber()
-//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected void _save(DataPacket dp) {
+         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
+         StreamSerializationUtils.write_to_file(str_bytes, _filename);
 
-//       void myHandleChanges(object newValue)
-//       {
-//          DataPacket dp = newValue as DataPacket;
-//          mySave(dp);
-//          myFireEvent(newValue);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
-
-//       protected void mySave(DataPacket dp)
-//       {
-//          for (int i = 0; i < 5 ; i++)
-//             try
-//             {
-//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
-//                {
-//                   BinaryFormatter bf = new BinaryFormatter();
-//                   BlueTable t = new BlueTable(dp.schema);
-//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
-//                      t.addRow(new BlueRow(r));
-//                   bf.Serialize(fs, t);
-//                }
-//                break;
-//             }
-//             catch (IOException)
-//             {
-//                System.Threading.Thread.Sleep(5);
-//             }
-//       }
-//    }
-// }
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -1,115 +1,115 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
 
 
-namespace BCG {
-   public class SnapshotStream : ViewStream {
+// namespace BCG {
+//    public class SnapshotStream : ViewStream {
 
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      public DateTime _snapshot_time;
-      public string _snapshot_file;
-      public IEnumerable<PlaybackPacket> _packets;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       public DateTime _snapshot_time;
+//       public string _snapshot_file;
+//       public IEnumerable<PlaybackPacket> _packets;
 
 
-      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-         string token_directory_name = $"{directory_name}\\{token}";
-         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-      }
+//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+//       }
 
-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-         _packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
 
-      private void _absorb_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-            } else {
-               bts.Log.assert(false, $"Operator: {op} is not expected");
-            }
-         }
-      }
+//       private void _absorb_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//             } else {
+//                bts.Log.assert(false, $"Operator: {op} is not expected");
+//             }
+//          }
+//       }
 
-      protected override void myOnLastSubscriber() {
-      }
+//       protected override void myOnLastSubscriber() {
+//       }
 
-      // public event SnapshotStream.Progress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
+//       // public event SnapshotStream.Progress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          return null;
+//       }
 
-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
 
-         }
-      }
-   }
-}
+//          }
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs

@@ -1,37 +1,66 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using System.Text.RegularExpressions;
 
 using BCG.Utilities.General;
 
-namespace BCG
-{
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
-   {
+namespace BCG {
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
+
+      public static TimeSpan parse_duration(string input) {
+         int days = 0, hours = 0, minutes = 0, seconds = 0;
+
+         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
+         Regex regex = new Regex(@"(\d+)([dhms])");
+         MatchCollection matches = regex.Matches(input);
+
+         foreach (Match match in matches) {
+            int value = int.Parse(match.Groups[1].Value);
+            string unit = match.Groups[2].Value;
+
+            switch (unit) {
+               case "d":
+                  days = value;
+                  break;
+               case "h":
+                  hours = value;
+                  break;
+               case "m":
+                  minutes = value;
+                  break;
+               case "s":
+                  seconds = value;
+                  break;
+            }
+         }
+
+         return new TimeSpan(days, hours, minutes, seconds);
+      }
+
       #region constructors
+
+      public TimeBasedFilter(string col_name, string row_time_to_live)
+      : this(col_name, parse_duration(row_time_to_live)) {
+
+      }
+
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
       public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
       }
 
       public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
       }
 
-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      {
-         myVsd               = vsd;
-         myRowTimeColumnName = rowTimeColumnName;
-         myRowTimeToLive     = rowTimeToLive;
-         myFilterFrequency   = filterFrequency;
-         myNextFilter        = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+         myVsd = vsd;
+         _time_col = rowTimeColumnName;
+         _ttl = rowTimeToLive;
+         myFilterFrequency = filterFrequency;
+         myNextFilter = DateTime.MinValue;
 
          // add worker
          myVsd.addWorker(this);
@@ -41,32 +70,25 @@ namespace BCG
       public event FilterChangeDelegate onFilterChange;
 
       #region IStreamFilter
-      public bool filter(IBlueRow row)
-      {
-         DateTime now       = DateTime.Now;
-         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
+      public bool filter(IBlueRow row) {
+         DateTime now = DateTime.Now;
 
-         if (now > timeStamp)
-         {
-            TimeSpan span = now - timeStamp;
-
-            if (span >= myRowTimeToLive)
-            {
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
+            if (span >= _ttl) {
                return (false);
             }
          }
-
          return (true);
       }
       #endregion
 
       #region IWorker
-      public void doWork()
-      {
+      public void doWork() {
          DateTime now = DateTime.Now;
 
-         if (now > myNextFilter)
-         {
+         if (now > myNextFilter) {
             // fire event
             onFilterChange();
 
@@ -76,8 +98,7 @@ namespace BCG
       #endregion
 
       #region IDisposable
-      public void Dispose()
-      {
+      public void Dispose() {
          // remove worker
          myVsd.removeWorker(this);
       }
@@ -85,10 +106,20 @@ namespace BCG
 
       #region state
       private ViewStreamDatabase myVsd;
-      private string             myRowTimeColumnName;
-      private TimeSpan           myRowTimeToLive;
-      private TimeSpan           myFilterFrequency;
-      private DateTime           myNextFilter;
+      private string _time_col;
+      private TimeSpan _ttl;
+      private TimeSpan myFilterFrequency;
+      private DateTime myNextFilter;
       #endregion
    }
+   [DataStreamUserRespresentation("timefilter")]
+   public class TimeBasedFilterStream : ViewStream {
+      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
+      :
+      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
+      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
+         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
+         mySetRealSource(new FilterStream(base_stream, filter));
+      }
+   }
 }

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -4,18 +4,14 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace BCG
-{
-   public class GroupByStream : MethodDrivenStream
-   {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator
-      {
+namespace BCG {
+   public class GroupByStream : MethodDrivenStream {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
          private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;            
+            _groupby_func = groupby_func;
          }
          public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
 
@@ -29,7 +25,7 @@ namespace BCG
       /// Serves as  a quick way to access all rows belonging to the same group
       /// </summary>
       private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable ;
+      private RowDictionaryTable myBookKeepingTable;
       private IGroupByEvaluator myGroupByEvaluator;
       private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
       //can map a schema to another
@@ -37,112 +33,98 @@ namespace BCG
       private int myIdGenerator = 0;
 
       public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
-      { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this (stream, groupByEvaluator, true)
-      {}
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this(stream, groupByEvaluator, true) { }
 
-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base (stream)
-      {
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base(stream) {
          myGroupByEvaluator = groupbyEvaluator;
          myIdGuard = idGuard;
       }
 
-      protected override void myHandleFirstCall ()
-      {
+      protected override void myHandleFirstCall() {
          mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
-         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
+         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
+         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
             myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
-         {
-            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
-               rowGroup, new BlueTable (mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
             if (newRow != null)
-               temp.addRow (newRow, myGetGroupString (rowGroup));
+               temp.addRow(newRow, myGetGroupString(rowGroup));
          }
-         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
+         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
       }
 
       /// <summary>
       /// Adding need not to anything as deletion already takes care
       /// of addition
       /// </summary>
-      protected override void myHandleIncrementalAdd()   {}
-      
+      protected override void myHandleIncrementalAdd() { }
+
       /// <summary>
       /// Both deletes and adds are handled here to make the intermediate results not
       /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
       /// because of the nature of it)
       /// </summary>
-      protected override void myHandleIncrementalDelete()   
-      {
+      protected override void myHandleIncrementalDelete() {
          myFireSwitch = false;
          RowDictionaryTable delGroups, addGroups;
          //form groups for deleted and added rows
-         myInitIncrementalChangeData (out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList ();
+         myInitIncrementalChangeData(out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList();
          string groupRepString;
 
-         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
             // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString (delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
-            handledAddGroups.Add (addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
-            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow (newRow, groupRepString);           
+            groupRepString = myGetGroupString(delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            handledAddGroups.Add(addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow(newRow, groupRepString);
          }
 
-         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
             // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains (addGroup))
-            {
+            if (!handledAddGroups.Contains(addGroup)) {
                //this is the case when there are rows added to a group and none
                //have been removed, hence it 'escapes' from being dealt with in 
                //the loop above
 
-               groupRepString = myGetGroupString (addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
+               groupRepString = myGetGroupString(addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
                // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
-                  new BlueTable (aggGroup.schema)));
-               myFireSwitch |= myMergeRow (newRow, groupRepString);
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
+               myFireSwitch |= myMergeRow(newRow, groupRepString);
             }
          }
       }
 
-      private bool myMergeRow (IBlueRow newRow, string groupRepString)
-      {
-         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
          //BlueEventLog.logAssert (newRow != null || existingRow != null);
          if (existingRow == null && newRow == null)
             return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
             return false;
-         else if (existingRow == null)
-         {
+         else if (existingRow == null) {
             //row added for the first time
-            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             return true;
-         }
-         else
-         {
-            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
-            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
-            if (newRow != null)
-            {
-               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+         } else {
+            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
+            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
+            if (newRow != null) {
+               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
             }
             return true;
          }
@@ -154,77 +136,68 @@ namespace BCG
       /// </summary>
       /// <param name="delGroups"></param>
       /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
-      {
+      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
          delGroups =
-            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
          addGroups =
-            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
          foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow (deletedRow);
+            myBookKeepingTable.removeRow(deletedRow);
          foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow (addedRow);
-         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
+            myBookKeepingTable.addRow(addedRow);
+         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          if (myBookKeepingTable.Count == 0)
             return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
-         myBookKeepingTable.clear ();
-         myGroupByEvaluator.clear ();
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         myBookKeepingTable.clear();
+         myGroupByEvaluator.clear();
       }
 
-      private IBlueRow myEvaluate (DataPacket dp)
-      {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
-         return evalRow == null ? null : 
+      private IBlueRow myEvaluate(DataPacket dp) {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
+         return evalRow == null ? null :
             myIdGuard ?
-            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
       }
 
-      private string myGetGroupString (IBlueRowCollection group)
-      {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString(IBlueRowCollection group) {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
+      protected override void myOnLastSubscriber() {
          //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
          //gets resubscribed, we start the process of building data again, and treat the callback as the first 
          //callback.
-         myInit ();
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myInit();
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myInit ()
-      {
-         base.myInit ();
-         if (myBookKeepingTable != null)
-         {
+      protected override void myInit() {
+         base.myInit();
+         if (myBookKeepingTable != null) {
             //to avoid calling init before the first callback
-            myBookKeepingTable.clear ();
-            myGroupByEvaluator.clear ();
+            myBookKeepingTable.clear();
+            myGroupByEvaluator.clear();
          }
       }
 
-      public static string getRepString (DataPacket dp, string[] colNames)
-      {
-         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
+      public static string getRepString(DataPacket dp, string[] colNames) {
+         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow (DataPacket dp)
-      {
+      public static IBlueRow getRepRow(DataPacket dp) {
          if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.newRows, 0);
+            return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.addedRows, 0);
+            return BlueDataUtility.extractRow(dp.addedRows, 0);
          else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.deletedRows, 0);
+            return BlueDataUtility.extractRow(dp.deletedRows, 0);
          // BlueEventLog.logAssert (false, "Illegal Data Packet");
          return null;
       }
@@ -237,22 +210,21 @@ namespace BCG
       /// <param name="newValue"></param>
       /// <param name="isAdd"></param>
       /// <returns></returns>
-      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
-      {
-         if (lastValue == null || lastValue.Equals (DBNull.Value))
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+         if (lastValue == null || lastValue.Equals(DBNull.Value))
             return newValue;
          // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
          Object retVal = null;
-         if (newValue.GetType().Equals (typeof(int)))
+         if (newValue.GetType().Equals(typeof(int)))
             retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals (typeof(Double)))
+         else if (newValue.GetType().Equals(typeof(Double)))
             retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals (typeof(Decimal)))
+         else if (lastValue.GetType().Equals(typeof(Decimal)))
             retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else{}
-            // BlueEventLog.logAssert (false, "Illegal type");
+         else { }
+         // BlueEventLog.logAssert (false, "Illegal type");
          return retVal;
       }
 

src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+    using System.Collections.Generic;
+
+    public class KeyValueToRelationalStream : ViewStream {
+        private readonly ViewStream _kv_str;
+        private readonly string _doc_id_col, _key_col, _val_col;
+        public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
+            _kv_str = kv_str;
+            _doc_id_col = doc_id_col;
+            _key_col = key_col;
+            _val_col = val_col;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _kv_str.onChange += _on_str_update;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _kv_str.onChange -= _on_str_update;
+        }
+
+        private void _on_str_update(object dp_obj) {
+            DataPacket dp = dp_obj as DataPacket;
+            var rows = dp.newRows;
+            var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
+            var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs

@@ -1,51 +1,64 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
+
 using BCG;
 using BCG.Utilities;
 
-namespace BCG
-{
-   public class LiteralStream : ViewStream
-   {
-      public LiteralStream(string tableName, string schema, string value)
-      {
-         string [] columns = schema.Split(' ');
+namespace BCG {
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');
 
 
-         ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id",0,typeof(int));
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
          int n = 1;
-         foreach (string c in columns)
-         {
-            string [] columnAndType = c.Split('/');
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
             if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c,n,typeof(double));
+               ci[n] = new ColumnInfo(c, n, typeof(double));
             else
-               ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
             n++;
          }
 
 
-         BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
-         string [] rows = value.Split(';');
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
          n = 0;
-         foreach (string r in rows) 
-            try 
-            {
-               if(r == string.Empty)
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
                   continue;
                t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));   
-            }
-            catch (Exception e)
-            {
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
                throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
             }
 
          mySetRealSource(new BlueRowCollectionStaticStream(t));
       }
 
-      public LiteralStream(string schema, string value) : this ("literal",schema, value)
-      {}
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+
+      // public LiteralStream(string value) {
+
+      // }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs

@@ -1,24 +1,24 @@
-using System;
+// using System;
 
-namespace BCG
-{
-   public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
-   {
-      IBlueTableSchema mySchema;
+// namespace BCG
+// {
+//    public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
+//    {
+//       IBlueTableSchema mySchema;
 
-      public IBlueRow map (IBlueRow row)
-      {
-         if (mySchema == null)
-         {
-            ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
-            int i = 0;
-            foreach (ColumnInfo ci in row.schema.columnInfo)
-               cis[i++] = ci.colType == null ?
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
-            mySchema = new BlueTableSchema (row.schema.tableName, cis);
-         }
-         return new BlueRow (mySchema, row);
-      }
-   }
-}
+//       public IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//          {
+//             ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
+//             int i = 0;
+//             foreach (ColumnInfo ci in row.schema.columnInfo)
+//                cis[i++] = ci.colType == null ?
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
+//             mySchema = new BlueTableSchema (row.schema.tableName, cis);
+//          }
+//          return new BlueRow (mySchema, row);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs

@@ -8,8 +8,7 @@ using mbts;
 using mbts.spark;
 using mbts.config;
 
-namespace BCG
-{
+namespace BCG {
 
 
    /// <summary>
@@ -18,27 +17,48 @@ namespace BCG
    /// desired row ie newRow = f(row), where f is the mapping fucntion. Note that the id 
    /// is preserved across the rows, ie row["id"] = f(row)["id"]
    /// </summary>
-   [DataStreamUserRespresentation (false)]
-   public class MapStream : MethodDrivenStream
-   {
-      private HighPerformanceTimer t = new HighPerformanceTimer ();
+   [DataStreamUserRespresentation(false)]
+   public class MapStream : MethodDrivenStream {
+      private HighPerformanceTimer t = new HighPerformanceTimer();
       protected IStreamMapper myMapper;
       protected IChangeableStreamMapper myChangeableMapper = null;
       private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
 
-      public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
-      {
-         if (myUseNewMap)
-            mySetRealSource (new SuperMapStream (vs, mapper));
-         else
-            myMapper = mapper;
+      public class FuncWrapper : IStreamMapper {
+         private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
+         Func<IBlueRow, IBlueRow> _row_map_func;
+
+         public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+         Func<IBlueRow, IBlueRow> row_map_func) {
+            _schema_map_func = schema_map_func;
+            _row_map_func = row_map_func;
+         }
+         public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+            return _schema_map_func(input_schema);
+         }
+         public IBlueRow map(IBlueRow row) {
+            return _row_map_func(row);
+         }
+
       }
-      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
-      {
+
+      public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
          if (myUseNewMap)
             mySetRealSource(new SuperMapStream(vs, mapper));
          else
-         {
+            myMapper = mapper;
+      }
+
+      public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+                                       Func<IBlueRow, IBlueRow> row_map_func)
+                                       :
+                                       this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
+      }
+
+      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
+         if (myUseNewMap)
+            mySetRealSource(new SuperMapStream(vs, mapper));
+         else {
             mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
             myChangeableMapper = mapper;
          }
@@ -47,77 +67,68 @@ namespace BCG
       /// <summary>
       /// Gets or sets the value of the mapper. Setting the value will result in re
       /// </summary>
-      private void myMapperChanged () 
-      {
+      private void myMapperChanged() {
          if (myLastArrivedPacket != null)
-            myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, 
+            myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows,
                                              myLastArrivedPacket.newRows));
       }
 
 
-      protected override void myHandleFirstCall ()
-      {
-         
-         IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
-         myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new BlueTable (outputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
+      protected override void myHandleFirstCall() {
+
+         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
+         myLastFiredPacket = new DataPacket();
+         myLastFiredPacket.newRows = new BlueTable(outputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
          foreach (IBlueRow row in myLastArrivedPacket.newRows)
-            ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
+            ((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
          myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
       }
 
-      protected override void myHandleIncrementalAdd()
-      {
+      protected override void myHandleIncrementalAdd() {
          //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows)
-         {
-            IBlueRow newRow = myMapper.map (row);
-            (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            IBlueRow newRow = myMapper.map(row);
+            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
+            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
          //System.Diagnostics.Trace.WriteLine (" ");
       }
 
-      protected override void myHandleIncrementalDelete()
-      {  
+      protected override void myHandleIncrementalDelete() {
          //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
-         {
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
             IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
+            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
+            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
          }
          //t.stop ();
          //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
       }
-     
 
-      protected override void myOnFirstSubscriber() 
-      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+
+      protected override void myOnFirstSubscriber() {
+         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override  void myOnLastSubscriber () 
-      {
-         myInit ();
+      protected override void myOnLastSubscriber() {
+         myInit();
          if (myStream != null)
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
       }
 
-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-         myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
       }
 
-      
+
    }
 
    /*

src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs

@@ -1,12 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+
 using BCG.Utilities;
 using BCG.Utilities.General;
+
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// SuperMap gives the user the ability
    /// to map one row to one or more rows
@@ -17,13 +18,11 @@ namespace BCG
    /// can chage itself and raise an event causing
    /// a remapping to happen
    /// </summary>
-   public class SuperMapStream : ViewStream
-   {
+   public class SuperMapStream : ViewStream {
       #region super mapper that wraps around stream mapper
-      private class MyMappingWrapper : ISuperStreamMapper
-      {
+      private class MyMappingWrapper : ISuperStreamMapper {
          private IStreamMapper myMapper;
-         
+
          public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
 
          #region ISuperStreamMapper Members
@@ -32,8 +31,7 @@ namespace BCG
 
          public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
 
-         public event MapperChangeDelegate onMapperChange
-         {
+         public event MapperChangeDelegate onMapperChange {
             add { }
             remove { }
          }
@@ -42,21 +40,18 @@ namespace BCG
       }
       #endregion
 
-      private class MyFuncWrapper : ISuperStreamMapper
-      {
+      private class MyFuncWrapper : ISuperStreamMapper {
          private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
          private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
          public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
-                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         {
+                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
             _get_schema = get_schema;
             _map_rows = map_rows;
          }
 
          public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
 
-         public IBlueRow[] map(IBlueRow r)
-         {
+         public IBlueRow[] map(IBlueRow r) {
             IEnumerable<IBlueRow> rows = _map_rows(r);
             return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
          }
@@ -76,19 +71,15 @@ namespace BCG
 
       public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
                                             Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, new MyFuncWrapper(get_schema, map_rows))
-      { }
+         : this(vs, new MyFuncWrapper(get_schema, map_rows)) { }
 
       public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, s => out_schema, map_rows)
-      { }
+         : this(vs, s => out_schema, map_rows) { }
 
       public SuperMapStream(ViewStream vs, IStreamMapper mapper)
-         : this(vs, new MyMappingWrapper(mapper))
-      { }
+         : this(vs, new MyMappingWrapper(mapper)) { }
 
-      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
-      {
+      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
          myStream = vs;
          myMapper = mapper;
          myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -96,8 +87,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          myStream.onChange -= new ChangeDelegate(myHandleData);
          myRows = null;
          myBaseRows = null;
@@ -105,8 +95,7 @@ namespace BCG
          //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
       }
 
-      private void myHandleData(object newValue)
-      {
+      private void myHandleData(object newValue) {
          DataPacket idp = newValue as DataPacket;
          //HighPerformanceTimer timer = new HighPerformanceTimer();
          //timer.start();
@@ -117,24 +106,19 @@ namespace BCG
          BlueTable dels = new BlueTable(myRows.schema),
                      adds = new BlueTable(myRows.schema);
 
-         foreach (IBlueRow r in dp.deletedRows)
-         {
+         foreach (IBlueRow r in dp.deletedRows) {
             List<IBlueRow> mappedRows = myMap[(int)r["id"]];
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
                dels.addRow(mr);
                myRows.deleteRow(mr);
             }
             myMap.Remove((int)r["id"]);
          }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
             IBlueRow[] mappedRows = myMapper.map(r);
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
                IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"]))
-               {
+               if (!myMap.ContainsKey((int)r["id"])) {
                   myMap.Add((int)r["id"], new List<IBlueRow>());
                }
                myMap[(int)r["id"]].Add(idChangedMapRow);
@@ -145,13 +129,12 @@ namespace BCG
          //timer.stop();
          // if (timer.lastIntervalTime > 20)
          //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
-         // myFireEvent(new DataPacket(myRows, adds, dels));
+         myFireEvent(new DataPacket(myRows, adds, dels));
       }
 
-      private void myHandleMapChange()
-      {
+      private void myHandleMapChange() {
          if (myBaseRows != null)
-            myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
+            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs

@@ -0,0 +1,142 @@
+using BCG;
+using System;
+using System.IO;
+using MongoDB.Bson;
+using BCG.Utilities.General;
+using System.Linq;
+using System.Collections.Generic;
+using System.Runtime.Intrinsics.X86;
+
+
+namespace bts {
+   // listens to the underlying stream, and writes each seriialized packet in a 
+   // seperate file
+   [DataStreamUserRespresentation("dp_recorder")]
+   public class DataPacketRecorderStream : ViewStream {
+      private ViewStream _base_stream;
+      private readonly string _folder;
+      StreamIOSerializationController _serializer = new StreamIOSerializationController();
+      private UInt32 _n_updates = 0;
+      public DataPacketRecorderStream(ViewStream base_stream, string folder) {
+         _base_stream = base_stream;
+         _folder = folder;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _base_stream.onChange += _handle_update;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _base_stream.onChange -= _handle_update;
+      }
+
+      private void _handle_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         byte[] ser_data = null;
+         if (_n_updates == 0) {
+            StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+            ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
+         } else {
+            StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
+            ser_data = _serializer.serialize(new StreamIOMessage(mra));
+         }
+         Log.assert(ser_data != null);
+         _n_updates++;
+         string file_name = $"{_folder}\\{_n_updates}.bin";
+         _write_to_file(ser_data, file_name);
+         myFireEvent(dp);
+      }
+
+      private static void _write_to_file(byte[] data, string file_name) {
+         using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
+            // Write the byte array to the file
+            fs.Write(data, 0, data.Length);
+         }
+      }
+   }
+   [DataStreamUserRespresentation("dp_replay")]
+   public class DataPacketReplayStream : ViewStream, IWorker {
+      private ViewStreamDatabase _vsd;
+      private readonly string _folder_path;
+      private readonly TimeSpan _fire_period;
+      IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+      private int _file_ptr = 0;
+      private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
+      private BlueTable _state = null;
+
+      public DataPacketReplayStream(string folder_path, string fire_period)
+      :
+      this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
+      }
+
+      public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
+         _vsd = vsd;
+         _folder_path = folder_path;
+         _fire_period = fire_period;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         MasterWorker.getInstance(false).addWorker(this, _fire_period);
+         _load_all_files();
+      }
+
+      protected override void myOnLastSubscriber() {
+         MasterWorker.getInstance(false).removeWorker(this);
+      }
+
+      public void doWork() {
+         if (_file_ptr >= _file_data.Count())
+            // we can set _file_ptr to 0 and keep going as well
+            return;
+         var curr_d = _file_data[_file_ptr++];
+         var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
+         StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
+         Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
+         switch (msg.op) {
+            case StreamIOMessage.StreamOp.CREATE: {
+                  _state = new BlueTable(msg.create_message.rows);
+                  break;
+               }
+            case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
+                  var mra = msg.multiple_remove_add_message;
+                  foreach (IBlueRow r in mra.removed_rows)
+                     _state.deleteRow(r);
+                  foreach (IBlueRow r in mra.added_rows)
+                     _state.addRow(r);
+                  break;
+               }
+         }
+         var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
+         myFireEvent(dp);
+      }
+
+      private void _load_all_files() {
+         Func<string, string> extract_file_name = (string full_path) => {
+            string fn = full_path.Split("\\").Last();
+            fn = fn.Split(".").First();
+            return fn;
+
+         };
+         string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
+
+         // Iterate over the .bin files and perform an action
+         foreach (string file in bin_files) {
+            byte[] file_data = File.ReadAllBytes(file);
+            _file_data.Add(Tuple.Create(file, file_data));
+         }
+         _file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
+            string fn1 = extract_file_name(t1.Item1);
+            string fn2 = extract_file_name(t2.Item1);
+            int fn_i1 = Int32.Parse(fn1);
+            int fn_i2 = Int32.Parse(fn2);
+            if (fn_i1 < fn_i2)
+               return -1;
+            else if (fn_i1 == fn_i2)
+               return 0;
+            else if (fn_i1 > fn_i2)
+               return 1;
+            return 0;
+         });
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -0,0 +1,195 @@
+using BCG;
+using BCG.Utilities.General;
+using System.Collections.Generic;
+
+//using MongoDB.Driver;
+
+using System;
+using System.IO;
+using System.Linq;
+
+namespace bts {
+    public class PubSubTestingStream : ViewStream, IWorker {
+        public class PayloadMessage {
+            public int msg_id = 0;
+            public string guid;
+            public Int64 travel_epoch_ms = 0;
+            public byte[] payload = null;
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+                msg_id = msg_id_args;
+                travel_epoch_ms = send_epoch_args;
+                payload = new byte[payload_size];
+                for (int i = 0; i < payload_size; i++) {
+                    payload[i] = payload_byte;
+                }
+            }
+            private PayloadMessage() { }
+            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+
+            public byte[] serialize() {
+                var str = new MemoryStream();
+                BinaryWriter bw = new BinaryWriter(str);
+                bw.Write(msg_id);
+                bw.Write(guid);
+                bw.Write(travel_epoch_ms);
+                bw.Write(payload.Length);
+                bw.Write(payload);
+                str.Position = 0;
+                var bytes = str.ToArray();
+                return bytes;
+            }
+
+            public static PayloadMessage deserialize(byte[] ser_bytes) {
+                var str = new MemoryStream(ser_bytes);
+                BinaryReader br = new BinaryReader(str);
+                int msg_id = br.ReadInt32();
+                string guid = br.ReadString();
+                Int64 epoch = br.ReadInt64();
+                int len = br.ReadInt32();
+                var payload = br.ReadBytes(len);
+                return new PayloadMessage() {
+                    msg_id = msg_id,
+                    guid = guid,
+                    travel_epoch_ms = epoch,
+                    payload = payload
+                };
+            }
+
+        }
+        //public void
+        private readonly int _msg_size;
+        private readonly TimeSpan _publish_period;
+        private DateTime _last_publish_time = DateTime.MinValue;
+        private PublisherCore _pub_core = new PublisherCore();
+        private BlueTable _table;
+        private int _n_pubs = 0;
+        private readonly int _max_pubs;
+        private int _n_updates = 0;
+        private bool _continue_sub_thread = true;
+        private ViewStreamDatabase _vsd;
+        private bool _fire_dirty = false;
+        private PayloadMessage _last_sent_msg, _last_recvd_msg;
+        private readonly int _payload_sz;
+        private const byte _payload_byte = (byte)'A';
+        private readonly string _tag_name;
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+            _vsd = vsd;
+            _payload_sz = payload_size;
+            var schema = BlueDataUtility.form_schema("PubSubTest",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("n_pubs", TypeCode.Int32),
+            Tuple.Create("n_recvs", TypeCode.Int32),
+            Tuple.Create("last_sent_id", TypeCode.Int32),
+            Tuple.Create("last_recv_id", TypeCode.Int32),
+            Tuple.Create("last_sent_guid", TypeCode.String),
+            Tuple.Create("last_recv_guid", TypeCode.String),
+            Tuple.Create("match", TypeCode.Boolean),
+            Tuple.Create("echo_time", TypeCode.Int32));
+            _table = new BlueTable(schema);
+            _publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
+            _msg_size = payload_size;
+            _max_pubs = max_pubs;
+            _tag_name = tag_name;
+        }
+
+        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
+        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
+
+        protected override void myOnFirstSubscriber() {
+            _vsd.addWorker(this);
+            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
+            th.IsBackground = true;
+            th.Start();
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vsd.removeWorker(this);
+            _continue_sub_thread = false;
+        }
+
+        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+            var idx = 0;
+            foreach (var msg in updates) {
+                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
+                idx++;
+            }
+        }
+
+        private void _subscribe_thread_func() {
+            // subscription is done in another thread
+            SubscriberCore sub_core = new SubscriberCore();
+            sub_core.subscribe(_tag_name);
+            while (_continue_sub_thread) {
+                // TODO recv_id
+                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
+                if (updates == null || updates.Count() == 0)
+                    continue;
+                _log_updates(updates);
+                _n_updates += updates.Count();
+                if (updates.Count() > 0) {
+                    var server_msg = updates.Last();
+                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
+                        _last_recvd_msg.travel_epoch_ms = epoch_now();
+                    } else {
+                        // why are we getting empty or null data ? (server is sending data regardless?)
+                    }
+                    _fire_dirty = true;
+                }
+            }
+        }
+
+        private void _publish_next_packet() {
+            if (_last_sent_msg == null) {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+                    guid = Guid.NewGuid().ToString()
+                };
+            } else {
+                _last_sent_msg.msg_id++;
+                _last_sent_msg.guid = Guid.NewGuid().ToString();
+                _last_sent_msg.travel_epoch_ms = epoch_now();
+            }
+            var ser_bytes = _last_sent_msg.serialize();
+            _n_pubs++;
+            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+        }
+
+        private void _fire_dp() {
+            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
+            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
+            object echo_time = DBNull.Value;
+            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
+            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
+            if (_last_sent_msg != null && _last_recvd_msg != null) {
+                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
+            }
+            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
+            IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
+                                        last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
+            BlueTable table = new BlueTable(_table.schema);
+            table.addRow(nr);
+            var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        public void doWork() {
+            if (DateTime.Now - _last_publish_time > _publish_period)
+                if (_n_pubs <= _max_pubs) {
+                    _publish_next_packet();
+                    _last_publish_time = DateTime.Now;
+                }
+            // publish 
+            // TODO: send id
+            if (_fire_dirty) {
+                _fire_dirty = false;
+                _fire_dp();
+            }
+        }
+
+        public static long epoch_now() {
+            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs

@@ -0,0 +1,55 @@
+namespace bts {
+   using BCG;
+   using BCG.Utilities.General;
+
+   using System;
+
+   public class SimpleTCPEchoStream : ViewStream, IWorker {
+      private string _ip;
+      private int _port;
+      private SimpleTCPClient _tcp_client;
+      private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
+      Tuple.Create("id", TypeCode.Int32),
+      Tuple.Create("event", TypeCode.String));
+      private BlueTable _table;
+      private SyncQueue<string> _action_queue = new SyncQueue<string>();
+
+      public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) {
+         _ip = ip;
+         _port = port;
+         _table = new BlueTable(_schema);
+         _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms);
+      }
+
+      protected override void myOnFirstSubscriber() {
+         ViewStreamDatabase.instance.addWorker(this);
+         _tcp_client.register_listener((msg) => { _on_msg(msg); });
+         _tcp_client.start();
+      }
+
+      private void _on_msg(string msg) {
+         _action_queue.enqueue(msg);
+      }
+
+      public void doWork() {
+         if (_action_queue.getCount() == 0)
+            return;
+         BlueTable added_rows = new BlueTable(_schema);
+         var sz = _action_queue.getCount();
+         for (int i = 0; i < sz; i++) {
+            string msg = _action_queue.dequeue();
+            if (msg == null)
+               break;
+            IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
+            _table.addRow(nr);
+            added_rows.addRow(nr);
+         }
+         if (added_rows.Count == 0)
+            return;
+         DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
+         myFireEvent(dp);
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs

@@ -0,0 +1,77 @@
+using System.Net.Sockets;
+using System.Net;
+
+using BCG;
+using System;
+using System.Collections.Generic;
+using MongoDB.Bson;
+using BCG.Utilities.General;
+using System.Linq;
+
+
+namespace bts {
+   // this class, a little specialized, takes the first snapshot thats fired 
+   // from the base stream sorts them by a given column, and fires one row at a time 
+   public class SnapshotReplayStream : ViewStream, IWorker {
+
+      private ViewStreamDatabase _vsd;
+      private ViewStream _snapshot_stream;
+      private readonly string _sort_col;
+      private readonly int _buffer_size;
+      private TimeSpan _fire_period;
+      private int _in_row_ptr = 0;
+      private List<IBlueRow> _in_rows = new List<IBlueRow>();
+      private CircularBufferTable _out_table;
+
+
+      public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
+      : this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
+
+      public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
+         _vsd = vsd;
+         _snapshot_stream = base_str;
+         _sort_col = sort_col;
+         _buffer_size = buffer_size;
+         _fire_period = fire_period;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _snapshot_stream.onChange += _handle_stream_update;
+         MasterWorker.getInstance(false).addWorker(this, _fire_period);
+      }
+
+      protected override void myOnLastSubscriber() {
+         base.myOnLastSubscriber();
+      }
+
+      private void _handle_stream_update(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         if (_in_rows.Count > 0)
+            return;
+         _out_table = new CircularBufferTable(dp.schema, _buffer_size);
+         // _in_rows is filled up only once
+         foreach (IBlueRow r in dp.newRows) {
+            _in_rows.Add(r);
+         }
+         _in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
+            return BlueDataUtility.compare_rows(r1, r2, _sort_col);
+         });
+      }
+
+      public void doWork() {
+         if (_in_row_ptr >= _in_rows.Count) {
+            // we can loop around or not here
+            return;
+         }
+         if (_in_rows.Count > 0) {
+            // fire row
+            IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
+            _out_table.add_row(curr_row);
+            DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
+               myLastValue == null ? null : myLastValue as DataPacket, false);
+            myFireEvent(dp);
+         }
+      }
+   }
+
+}

src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs

@@ -0,0 +1,817 @@
+
+
+using Amazon.Runtime.Internal;
+
+using bts;
+
+using DnsClient.Protocol;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System;
+using Microsoft.VisualBasic;
+using Spark.Core.Collections;
+using System.Reflection;
+
+namespace BCG {
+
+    /// <summary>
+    /// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
+    /// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
+    /// </summary>
+    [DataStreamUserRespresentation("tests")]
+    public class TesStream : ViewStream {
+        static byte[] create_byte_array() {
+            Random random = new Random();
+            var sz = random.Next(1, 1000);// % 1000;
+
+            // Generate a random byte
+            byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
+            byte[] rv = new byte[sz];
+            for (int i = 0; i < sz; i++) {
+                rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
+            }
+            return rv;
+        }
+
+        private IBlueTableSchema _schema;
+        class TestResult {
+            public string test_name = MethodBase.GetCurrentMethod().Name;
+            public bool test_result;
+        }
+        private List<Func<TestResult>> _tests = new
+         List<Func<TestResult>>();
+
+        public TesStream() {
+            _schema = BlueDataUtility.form_schema("Tests",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("test", TypeCode.String),
+            Tuple.Create("result", TypeCode.Boolean),
+            Tuple.Create("exception", TypeCode.String)
+            );
+            // register tests here
+            // _tests.Add(new Func<TestResult>(_run_sub_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_pub_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
+            // _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
+            // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
+            // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
+            //  _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
+            //_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
+            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
+            // _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
+            // _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
+            // _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
+            //_tests.Add(new Func<TestResult>(_cpp_data_test));
+            //_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
+            _tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
+
+        }
+
+        protected override void myOnFirstSubscriber() {
+            BlueTable rows = new BlueTable(_schema);
+            // u the tests here            
+            foreach (var test in _tests) {
+                string ex_msg = String.Empty;
+                TestResult tr = new TestResult() {
+                    test_name = test.Method.Name,
+                    test_result = false
+                };
+                try {
+                    tr = test();
+                } catch (Exception ex) {
+                    ex_msg = ex.Message + "\n" + ex.StackTrace;
+                }
+                IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
+                rows.addRow(r);
+            }
+            var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+        protected override void myOnLastSubscriber() {
+        }
+
+        private TestResult _run_sub_msg_test() {
+            bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
+                connection = Guid.NewGuid().ToString(),
+                subscribing_app = Guid.NewGuid().ToString(),
+                tags = new string[] { "one", "two", "three" }
+            };
+            MemoryStream str = new MemoryStream();
+            sub_msg.serialize_into_str(str);
+            str.Position = 0;
+            bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
+            sub_msg_.consruct_deserialize(str);
+            bool res = sub_msg.Equals(sub_msg_);
+            return new TestResult {
+                test_name = "SubscribeMsg",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pub_msg_test() {
+            var pub_msg = new bts.PublishMsg() {
+                connection = Guid.NewGuid().ToString(),
+                publishing_app = Guid.NewGuid().ToString(),
+                tag = Guid.NewGuid().ToString(),
+                tag_type = "DS_",
+                is_compressed = false,
+                publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                data_to_publish = create_byte_array(),
+            };
+            MemoryStream str = new MemoryStream();
+            pub_msg.serialize_into_str(str);
+            str.Position = 0;
+            var pub_msg_ = new bts.PublishMsg();
+            pub_msg_.consruct_deserialize(str);
+            bool res = pub_msg.Equals(pub_msg_);
+            return new TestResult {
+                test_name = "PublishMsg",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pubsub_server_msg_test() {
+            var msg = new bts.PubSubServerMessage() {
+                tag = Guid.NewGuid().ToString(),
+                tag_type = Guid.NewGuid().ToString(),
+                data = create_byte_array()
+            };
+            MemoryStream str = new MemoryStream();
+            msg.serialize_into_str(str);
+            str.Position = 0;
+            var msg_ = new bts.PubSubServerMessage();
+            msg_.consruct_deserialize(str);
+            bool res = msg.Equals(msg_);
+            return new TestResult {
+                test_name = "PubSubServerMessage",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_pubsub_client_msg_test() {
+            PubSubClientMessage msg = new PubSubClientMessage() {
+                msg_type = PubSubMessageType.SUBSCRIBE,
+                sub_msg = new bts.SubscribeMsg() {
+                    connection = Guid.NewGuid().ToString(),
+                    subscribing_app = Guid.NewGuid().ToString(),
+                    tags = new string[] { "one", "two", "three" }
+                }
+            };
+            var str = new MemoryStream();
+            msg.serialize_into_str(str);
+            str.Position = 0;
+            var msg_ = new PubSubClientMessage();
+            msg_.consruct_deserialize(str);
+            var res = msg.Equals(msg_);
+            return new TestResult {
+                test_name = "PubSubClientMessage",
+                test_result = res
+            };
+        }
+
+        private TestResult _run_columninfo_serialization_test() {
+            Func<ColumnInfo, bool> test_ci = (ci) => {
+                MemoryStream str = new MemoryStream();
+                BinaryWriter bw = new BinaryWriter(str);
+                StreamSerializationUtils.serialize_column_info(ci, ref bw);
+                str.Position = 0;
+                //var ci_txt = BlueDataUtility.to_string(ci);
+                //var ci_ser = 
+                //bts.Log.Gen.info("ci: {}")
+                BinaryReader br = new BinaryReader(str);
+                var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
+                return BlueDataUtility.compute_equality(ci, ci_prime);
+            };
+            int i = 0;
+            foreach (Type t in BlueDataUtility.valid_column_types()) {
+                ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
+                i++, t);
+                var res = test_ci(ci);
+                if (!res) {
+                    return new TestResult {
+                        test_name = "ColumnInfoSerialization",
+                        test_result = false
+                    };
+                }
+            }
+            return new TestResult {
+                test_name = "ColumnInfoSerialization",
+                test_result = true
+            };
+        }
+
+
+        private TestResult _run_schema_serialization_test() {
+            var table_name = "test_table";
+            var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
+            IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            StreamSerializationUtils.serialize_schema(schema, ref bw);
+            _write_to_file(str, @"c:\Temp\tests\schema.bin");
+            var hex_txt = StreamSerializationUtils.to_hex(str);
+            bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
+
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
+            var res = BlueDataUtility.compute_equality(schema, schema_prime);
+
+            return new TestResult {
+                test_name = "SchemaSerialization",
+                test_result = res
+            };
+        }
+
+
+
+
+        private TestResult _run_datetime_seriliaztion_test() {
+            DateTime dt = DateTime.Now;
+            DateTimeOffset dt_off = new DateTimeOffset(dt);
+            long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
+            DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
+            DateTime dt_p = dt_off_pr.DateTime;
+            var passes = dt.Equals(dt_p);
+            return new TestResult {
+                test_name = "_run_datetime_seriliaztion_test",
+                test_result = false
+            };
+        }
+
+        private TestResult _run_full_stream_serialization_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+            UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
+            ser.serialize(stream_io_msg, ref bw);
+            str.Position = 0;
+            BinaryReader br = new System.IO.BinaryReader(str);
+            UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
+            var stream_io_msg_prime = deser.deserialize(ref br);
+            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
+            var n_dels = 50;
+            BlueTable del_rows = new BlueTable(cr_rows.schema);
+            foreach (IBlueRow r in cr_rows) {
+                del_rows.addRow(r);
+                if (del_rows.Count == n_dels)
+                    break;
+            }
+            IBlueRowCollection added_rows = del_rows;
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            str = new MemoryStream();
+            bw = new BinaryWriter(str);
+            stream_io_msg = new bts.StreamIOMessage(mra_msg);
+            ser.serialize(stream_io_msg, ref bw);
+            bw.BaseStream.Position = 0;
+            br = new BinaryReader(str);
+            //deser = new UniversalStreamIODeserializer();
+            stream_io_msg_prime = deser.deserialize(ref br);
+            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
+            return new TestResult {
+                test_name = "UniversalSerialization",
+                test_result = passes
+            };
+        }
+
+        private TestResult _run_full_stream_serialization_with_controller_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
+            var ser_bytes = ser.serialize(stream_io_msg);
+            IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
+            var stream_io_msg_prime = deser.deserialize(ser_bytes);
+            bool passes = stream_io_msg.Equals(stream_io_msg_prime);
+            //mult remove adds
+            // remove all rows and add 50 more
+
+            BlueTable del_rows = new BlueTable(cr_rows);
+            BlueTable added_rows = new BlueTable(cr_rows.schema);
+            var n_adds = 50;
+            for (int i = 0; i < n_adds; i++) {
+                var nr = generate_random_row(schema, i);
+                added_rows.addRow(nr);
+            }
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            stream_io_msg = new bts.StreamIOMessage(mra_msg);
+            ser_bytes = ser.serialize(stream_io_msg);
+            var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
+            Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
+            stream_io_msg_prime = deser.deserialize(ser_bytes);
+            passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
+            return new TestResult {
+                test_name = "UniversalSerializationController",
+                test_result = passes
+            };
+        }
+
+        private TestResult _row_serialization_tests_no_dbnulls() {
+            // IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+            //                                                     new Tuple<string, TypeCode>("id", TypeCode.Int32),
+            //                                                     new Tuple<string, TypeCode>("i1", TypeCode.Int32),
+            //                                                     new Tuple<string, TypeCode>("d1", TypeCode.Double),
+            //                                                     new Tuple<string, TypeCode>("string_1", TypeCode.String));
+            // bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            // IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
+            var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            MemoryStream str = new MemoryStream();
+            IBlueRow r = generate_random_row(schema, 0, false);
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            _write_to_file(str, @"C:\Temp\tests\row.bin");
+            str.Position = 0;
+            byte[] raw_byte_array = str.ToArray();
+            var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
+            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            string row_str = r_prime.to_string();
+            Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_NonNull",
+                test_result = passes
+            };
+        }
+
+        private TestResult _deserialize_pub_msg_test() {
+            var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
+            PubSubClientMessage ps_client_msg = new PubSubClientMessage();
+            ps_client_msg.consruct_deserialize(pub_msg_ser);
+            bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
+            return new TestResult {
+                test_name = "_deserialize_pub_msg_test",
+                test_result = passes
+            };
+        }
+
+        private TestResult _cpp_data_test() {
+            //create and write schema
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
+                                                                );
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
+            StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+            StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+
+            IBlueTableSchema schema_p = null;
+            var schema_file = @"c:\Temp\tests\schema.bin";
+            var row_file = @"C:\Temp\tests\row.bin";
+            var create_op_file = @"C:\Temp\tests\create_op.bin";
+            var mra_op_file = @"C:\Temp\tests\mra_op.bin";
+            IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
+
+            //serialize schema, write it to a file, read it back, deserialize it and match schemas
+            byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
+            _write_to_file(ser_schema_bytes, schema_file);
+
+            byte[] schema_file_bytes = _read_from_file(schema_file);
+            schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
+            bool passes = BlueDataUtility.compute_equality(schema, schema_p);
+
+            //serialize row, write it to a file, read it back, deserialize it and match schemas    
+            byte[] ser_row_bytes = row_ser.serialize_row(row);
+            _write_to_file(ser_row_bytes, row_file);
+
+            byte[] row_file_bytes = _read_from_file(row_file);
+            var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
+            Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
+
+            IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
+            passes = passes && BlueDataUtility.compute_equality(row, row_p);
+
+            {
+                // serialize a create operation
+                int n_rows = 100;
+                BlueTable table = new BlueTable(schema);
+                for (int i = 0; i < n_rows; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
+                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
+                _write_to_file(create_ser_bytes, create_op_file);
+
+                // read it and make sure its right
+                byte[] create_op_file_bytes = _read_from_file(create_op_file);
+                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+                var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
+                _write_to_file(create_ser_bytes, mra_op_file);
+
+                passes = passes && cr_msg.Equals(cr_msg_p);
+
+            }
+            //multiple remove adds
+            {
+                // remove all rows and add them again
+                int n_rows = 100;
+                BlueTable table = new BlueTable(schema);
+                for (int i = 0; i < n_rows; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
+
+                //StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
+                _write_to_file(mra_ser_bytes, mra_op_file);
+
+                // reasd it and make sure it matches
+                byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
+                //StreamIODeserializationController deser_controller = new StreamIODeserializationController();
+                StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
+                passes = passes && mra_msg.Equals(mra_msg_p);
+            }
+            return new TestResult {
+                test_name = "_cpp_data_test",
+                test_result = passes
+            };
+        }
+
+        private TestResult _write_stream_updates_to_files() {
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d_1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String)
+                                                                );
+            var packets_path = @"C:\Temp\tests\stream_packets\";
+            Func<long> now_epochs = () => {
+                DateTimeOffset now = DateTimeOffset.UtcNow;
+                // Convert to Unix time in milliseconds
+                long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
+                return unix_time_milliseconds;
+            };
+            Func<int, bool, string> curr_file_path = (id, create) => {
+                //var epoch_ms = now_epochs();
+                var file_path = $"{packets_path}{id}.bin";
+                if (create)
+                    file_path = $"{packets_path}create_{id}.bin";
+                return file_path;
+            };
+            // write 1000 stream updates
+            {
+                int init_sz = 100;
+                BlueTable table = new BlueTable(schema);
+                StreamIOSerializationController ser_controller = new StreamIOSerializationController();
+                // create table of 100 rows
+                for (int i = 0; i < init_sz; i++) {
+                    IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
+                    table.addRow(nr);
+                }
+                // write create msg
+                StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
+                byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
+                var create_file_path = curr_file_path(table.Count, true);
+                _write_to_file(create_ser_bytes, create_file_path);
+
+                // add 1000 rows and write an io msg for each update and write it to a file
+                int n_updates = 1000;
+                int period_ms = 50;
+
+                // write a create
+                for (int i = 0; i < n_updates; i++) {
+                    // addition
+                    BlueTable adds = new BlueTable(schema);
+                    int id = table.Count;
+                    IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
+                    adds.addRow(nr);
+                    BlueTable dels = new BlueTable(schema);
+                    table.addRow(nr);
+                    StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
+                    byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
+                    var mra_file_path = curr_file_path(id, false);
+                    _write_to_file(mra_ser_bytes, mra_file_path);
+
+                    System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
+                }
+                // now start writing increments and add 1 row at a time
+            }
+            return new TestResult {
+                test_name = "_write_updates_as_files",
+                test_result = true
+            };
+        }
+
+
+        private TestResult _row_serialization_tests_with_dbnulls() {
+            IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
+                                                                new Tuple<string, TypeCode>("id", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("i1", TypeCode.Int32),
+                                                                new Tuple<string, TypeCode>("d1", TypeCode.Double),
+                                                                new Tuple<string, TypeCode>("string_1", TypeCode.String));
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_WithDBNulls",
+                test_result = passes
+            };
+        }
+
+        private TestResult _row_serialization_tests_all_types() {
+            var schema = create_schema_with_all_types();
+            bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
+            IBlueRow r = generate_random_row(schema, 0, false);
+            MemoryStream str = new MemoryStream();
+            BinaryWriter bw = new BinaryWriter(str);
+            row_ser.serialize_row(r, 0, ref bw);
+            _write_to_file(str, @"C:\Temp\tests\row.bin");
+            str.Position = 0;
+            BinaryReader br = new BinaryReader(str);
+            bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
+            IBlueRow r_prime = des.deserialize_row(ref br);
+            var passes = BlueDataUtility.compute_equality(r, r_prime);
+            return new TestResult {
+                test_name = "UniversalRowSerialization_AllTypes",
+                test_result = passes
+            };
+        }
+
+        private static TestResult _run_pubsub_server_stream_tag_handler_test() {
+            // create message
+            var schema = create_schema_with_all_types();
+            var tag = "DS_" + schema.tableName;
+            BlueTable cr_rows = new BlueTable(schema);
+            // 100 rows
+            int n_rows = 100;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                cr_rows.addRow(nr);
+            }
+
+            PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
+            bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
+            bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
+            IStreamIOMsgSerializer ser = new StreamIOSerializationController();
+            byte[] msg_bytes = ser.serialize(str_io_msg);
+            handler.update(tag, msg_bytes);
+
+            //now get it back for the handler
+            byte[] content = handler.get_content(tag);
+            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
+            bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
+            // test here
+            var passes = str_io_msg.Equals(str_io_msg_p);
+            // delete all rows
+            BlueTable del_rows = new BlueTable(cr_rows);
+            cr_rows.clear();
+            // added 50 new rows
+            BlueTable added_rows = new BlueTable(del_rows.schema);
+            int new_state_sz = 50;
+            for (int i = 0; i < new_state_sz; i++) {
+                var nr = generate_random_row(schema, i);
+                added_rows.addRow(nr);
+            }
+            cr_rows.addRows(added_rows);
+            // the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
+            // 1. generate op
+            bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+            str_io_msg = new bts.StreamIOMessage(mra);
+            // 2. serialize it into bytes
+            msg_bytes = ser.serialize(str_io_msg);
+            // 3. send it to handler
+            handler.update(tag, msg_bytes);
+            // 4. get the state from the handler
+            byte[] content_p = handler.get_content(tag);
+            // 5. deserialize the state into a stream op
+            str_io_msg_p = deser.deserialize(content_p);
+            // 6. check to make sure the state's rows are good
+            passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
+                    BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
+            return new TestResult {
+                test_result = passes,
+                test_name = "run_pubsub_server_stream_tag_handler_test"
+            };
+        }
+
+
+        private static TestResult _run_single_tag_data_test_clean() {
+            // create message
+            var test_name = MethodBase.GetCurrentMethod().Name;
+            var schema = create_schema_with_all_types();
+            var tag = "DS_" + schema.tableName;
+            BlueTable table = new BlueTable(schema);
+            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
+            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            // create with 100 rows
+            int n_rows = 100;
+            bool is_succ = true;
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                // perform the same action SingletagData, and verify the states match
+                PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
+                table.addRow(nr);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            // change all rows
+            for (int i = 0; i < n_rows; i++) {
+                var nr = generate_random_row(schema, i);
+                var dr = table[i];
+                table.deleteRow(dr);
+                table.addRow(nr);
+                PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            // delete all rows
+            var aux_table = new BlueTable(table);
+            foreach (IBlueRow r in aux_table) {
+                table.deleteRow(r);
+                PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
+                st_data.handle_publish(pub_msg);
+                var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
+                is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
+            }
+            return new TestResult() {
+                test_result = is_succ,
+                test_name = test_name
+            };
+        }
+
+
+
+        #region Helper functions
+
+        private static void _write_to_file(byte[] ser_bytes, string file_name) {
+            _write_to_file(new MemoryStream(ser_bytes), file_name);
+        }
+
+        private static void _write_to_file(MemoryStream str, string file_name) {
+            if (String.IsNullOrEmpty(file_name))
+                return;
+            byte[] str_bytes = str.ToArray();
+            using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
+                // Write the byte array to the file
+                fs.Write(str_bytes, 0, str_bytes.Length);
+            }
+        }
+
+        private static byte[] _read_from_file(string file_path) {
+            var rv = File.ReadAllBytes(file_path);
+            return rv;
+        }
+
+        public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
+            var content = st_data.get_current_content();
+            var msg_p = deser.deserialize(content);
+            Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
+            return msg_p.create_message.rows;
+
+        }
+
+        public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
+            ref IStreamIOMsgSerializer serializer) {
+            Func<IBlueTableSchema> get_schema = () => {
+                return (added_row != null ? added_row : removed_row).schema;
+            };
+            bts.StreamIOMessage str_io_msg = null;
+            if (is_create) {
+                Log.assert(added_row != null);
+                var aux_new = new BlueTable(added_row.schema);
+                aux_new.addRow(added_row);
+                var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
+                str_io_msg = new bts.StreamIOMessage(cr_msg);
+            } else {
+                var schema = get_schema();
+                BlueTable added_table = new BlueTable(schema);
+                if (added_row != null)
+                    added_table.addRow(added_row);
+                BlueTable rem_table = new BlueTable(schema);
+                if (removed_row != null)
+                    rem_table.addRow(removed_row);
+                bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
+                    new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
+                str_io_msg = new bts.StreamIOMessage(mra_msg);
+            }
+            byte[] msg_bytes = serializer.serialize(str_io_msg);
+            PublishMsg pub_msg = new PublishMsg() {
+                connection = "test_connection",
+                tag = $"DS_{get_schema()}",
+                tag_type = "DS",
+                data_to_publish = msg_bytes
+            };
+            return pub_msg;
+        }
+
+        public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
+            excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
+            var table_name = "test_table";
+            BlueTableSchema schema = new BlueTableSchema();
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            foreach (Type t in BlueDataUtility.valid_column_types()) {
+                if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
+                    continue;
+                var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
+                ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
+                cis.Add(ci);
+            }
+            cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
+            schema = new BlueTableSchema(table_name, cis.ToArray());
+            return schema;
+        }
+
+        public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
+            Random rnd = new Random();
+            object[] row_vals = new object[schema.columnInfo.Length];
+            foreach (ColumnInfo ci in schema.columnInfo) {
+                if (ci.colName == "id") {
+                    row_vals[ci.colNo] = id;
+                    continue;
+                }
+                object v = get_random_object(ci.type_code());
+                if (include_dbnulls && rnd.NextSingle() % 4 == 0)
+                    v = DBNull.Value;
+                row_vals[ci.colNo] = v;
+            }
+            return new BlueRow(schema, row_vals);
+        }
+
+        public static object get_random_object(TypeCode type_code) {
+            Random random = new Random();
+            switch (type_code) {
+                case TypeCode.Boolean:
+                    return random.Next(2) == 0; // Random boolean value (true or false)
+                case TypeCode.Byte:
+                    return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
+                case TypeCode.Char:
+                    return (char)random.Next(65, 65 + 27); // Random char value
+                case TypeCode.Decimal:
+                    Decimal dec = 100.123M;
+                    return dec;
+                case TypeCode.DateTime:
+                    return DateTime.Now;
+                case TypeCode.Double:
+                    return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
+                case TypeCode.Int16:
+                    return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
+                case TypeCode.Int32:
+                    return random.Next(int.MinValue, int.MaxValue); // Random int value
+                case TypeCode.Int64:
+                    return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
+                case TypeCode.SByte:
+                    return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
+                case TypeCode.Single:
+                    return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
+                case TypeCode.String:
+                    return Guid.NewGuid().ToString();
+                case TypeCode.UInt16:
+                    return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
+                case TypeCode.UInt32:
+                    return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
+                case TypeCode.UInt64:
+                    byte[] ulongBuffer = new byte[8];
+                    random.NextBytes(ulongBuffer);
+                    return BitConverter.ToUInt64(ulongBuffer, 0);
+                default:
+                    throw new ArgumentException($"Unsupported TypeCode: {type_code}");
+                    return null;
+            }
+        }
+        #endregion
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Security.Cryptography.X509Certificates;
+
+using BCG;
+using BCG.Utilities.General;
+
+using HtmlAgilityPack;
+
+namespace BCG {
+    class Program {
+        static void Main() {
+            string url = "http://quotes.toscrape.com";
+            List<string> quotes = ScrapeQuotes(url);
+
+            Console.WriteLine("Quotes from http://quotes.toscrape.com:");
+            foreach (var quote in quotes) {
+                Console.WriteLine(quote);
+            }
+        }
+
+        static List<string> ScrapeQuotes(string url) {
+            List<string> quotes = new List<string>();
+
+            // Create a new HtmlWeb instance
+            HtmlWeb web = new HtmlWeb();
+
+            try {
+                // Load the HTML document from the specified URL
+                HtmlDocument document = web.Load(url);
+
+                // Extract quotes from the HTML document
+                var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
+                if (quoteNodes != null) {
+                    foreach (var quoteNode in quoteNodes) {
+                        string quote = quoteNode.InnerText.Trim();
+                        quotes.Add(quote);
+                    }
+                }
+            } catch (Exception ex) {
+                Console.WriteLine($"Error: {ex.Message}");
+            }
+
+            return quotes;
+        }
+    }
+
+    public class WebScapperStream : ViewStream, IWorker {
+        private readonly string _url;
+        private readonly string _xpath;
+        private readonly ViewStreamDatabase _vsd;
+        private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
+        private readonly IBlueTableSchema _schema;
+
+        public WebScapperStream()
+        :
+        this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
+        public WebScapperStream(string url, string xpath)
+        : this(url, xpath, ViewStreamDatabase.instance) { }
+        public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
+            _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("url", TypeCode.UInt32),
+                                                Tuple.Create("line", TypeCode.String));
+            _url = url;
+            _xpath = xpath;
+            _vsd = vsd;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            var lines = scrape_lines(_url, _xpath);
+            Func<string, int> get_id = (line) => {
+                int rv = 0;
+                if (!_line_id_map.TryGetValue(line, out rv))
+                    _line_id_map.Add(line, _line_id_map.Count);
+                return rv;
+            };
+            BlueTable nrows = new BlueTable(_schema);
+            foreach (var l in lines) {
+                bts.Log.Gen.info($"line: {l}");
+                var id = get_id(l);
+                BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
+                nrows.addRow(nr);
+            }
+            var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
+            myFireEvent(dp);
+        }
+
+        static List<string> scrape_lines(string url, string xpath) {
+            List<string> quotes = new List<string>();
+            // Create a new HtmlWeb instance
+            HtmlWeb web = new HtmlWeb();
+            try {
+                // Load the HTML document from the specified URL
+                HtmlDocument document = web.Load(url);
+                // Extract quotes from the HTML document
+                var quote_nodes = document.DocumentNode.SelectNodes(xpath);
+                if (quote_nodes != null) {
+                    foreach (var qn in quote_nodes) {
+                        string quote = qn.InnerText.Trim();
+                        quotes.Add(quote);
+                    }
+                }
+            } catch (Exception ex) {
+                Console.WriteLine($"Error: {ex.Message}");
+            }
+
+            return quotes;
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/BenzingaNewStream.cs

@@ -0,0 +1,66 @@
+namespace bts {
+   using BCG;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Net;
+   using System.Text.Json;
+   using System.Text.Json.Nodes;
+   [DataStreamUserRespresentation("benzingaNews")]
+   public class BenzingaNewStream : ViewStream {
+      private IBlueTableSchema _schema = null;
+      private int _id = 0;
+      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
+      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
+         // Hiding body (with project, columns can be hidden, or renamed)
+         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
+         // add column news_source = 'Benzinga', so we can append other news sources
+         output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
+         // now SuperMapStream is used to map one row into multiple, one per symbol,
+         // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
+         // return an array of rows, per one row)
+         output_stream = new SuperMapStream(mongodb_raw_stream,
+
+        (IBlueTableSchema s) => {
+           // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
+           if (_schema == null) {
+              var out_schema = s;
+              out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
+              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
+              out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
+              out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
+              _schema = out_schema;
+           }
+           return _schema;
+        },
+        (IBlueRow r) => {
+           var json_objs = BlueDataUtility.json_to_json_node_list(r.get<string>("securities"));
+           List<IBlueRow> expanded_rows = new List<IBlueRow>();
+           if (json_objs.Count == 0) {
+              MutableBlueRow nr = new MutableBlueRow(_schema, r);
+              nr.setValue("id", _id++);
+              expanded_rows.Add(nr);
+           }
+           foreach (JsonNode jnode in json_objs) {
+              if (jnode is JsonObject json_object) {
+                 MutableBlueRow nr = new MutableBlueRow(_schema, r);
+                 nr.setValue("id", _id++);
+                 nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
+                 nr.setValue("symbol", jnode["symbol"]);
+                 nr.setValue("exchange", jnode["exchange"]);
+                 nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
+                 expanded_rows.Add(nr);
+              }
+           }
+           return expanded_rows;
+
+        });
+         // now we creaate a TimeFilter stream to keep the table as a circular buffer of the last N days
+         // passed A an argument in the constructor above
+         // time_filter is a string like 7d 1h 30 m 45s 
+         output_stream = new TimeBasedFilterStream(output_stream, "created_at", ttl);
+         mySetRealSource(output_stream);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -1,37 +1,256 @@
 using System;
-
+using System.Collections.Generic;
 using MongoDB.Driver;
 using MongoDB.Bson;
+using BCG;
+using BCG.Utilities.General;
+using BCG;
+using mbts.spark;
 
 
-class Program {
-    static void Main() {
-        // Set the connection string and database name
-        string connectionString = "mongodb://localhost:27017";
-        string databaseName = "your_database_name";
-        string collectionName = "your_collection_name";
+namespace bts {
+    [DataStreamUserRespresentation("mongodb_sql")]
+    public class MongoDBSQLStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        // make this a sorted table
+        private BlueTable _table;
 
-        // Create a MongoClient
-        MongoClient client = new MongoClient(connectionString);
 
         // Access the database
-        IMongoDatabase database = client.GetDatabase(databaseName);
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
 
-        // Access the collection
-        IMongoCollection<BsonDocument> collection = database.GetCollection<BsonDocument>(collectionName);
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        // Fetch all documents from the collection
-        var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+        }
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
 
-        // Iterate through the documents and print values
-        foreach (var document in documents.ToEnumerable()) {
-            // Assuming your document structure has fields named "Field1", "Field2", etc.
-            Console.WriteLine("Field1: " + document["Field1"]);
-            Console.WriteLine("Field2: " + document["Field2"]);
-            // Add more fields as needed
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+        }
 
-            // Print a separator between documents
-            Console.WriteLine(new string('-', 20));
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_row(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables: {timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+            foreach (var n in doc_row.Names) {
+                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+            }
+            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_row(BsonDocument doc) {
+            var key_val = doc["_id"];
+            var key = key_val.ToString();
+            int id;
+            if (!_id_map.TryGetValue(key, out id)) {
+                id = _id_map.Count;
+                _id_map.Add(key, id);
+            }
+            MutableBlueRow nr = new MutableBlueRow(_table.schema);
+            nr.setValue("id", id);
+            foreach (var n in doc.Names) {
+                var val = doc[n].ToString();
+                nr.setValue(n, val);
+            }
+            _table.addRow(nr);
         }
     }
-}
+
+    [DataStreamUserRespresentation("mongodb")]
+    public class MongoDBStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
+
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
+
+        public MongoDBStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+        }
+        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                //bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_doc(document);
+            }
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+            timer.start();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.stop();
+            bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
+            Tuple.Create("id", TypeCode.Int32),
+            Tuple.Create("doc_id", TypeCode.String),
+            Tuple.Create("key", TypeCode.String),
+            Tuple.Create("value", TypeCode.String));
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_doc(BsonDocument doc) {
+            var doc_id_bson = doc["_id"];
+            var doc_id = doc_id_bson.ToString();
+            Func<string, string, int> get_id = (string doc_id, string key) => {
+                string hash_key = $"{doc_id}_{key}";
+                int id = 0;
+                if (!_id_map.TryGetValue(hash_key, out id)) {
+                    id = _id_map.Count;
+                    _id_map.Add(hash_key, id);
+                }
+                return id;
+            };
+            foreach (var n in doc.Names) {
+                string key = n;
+                var id = get_id(doc_id, key);
+                MutableBlueRow nr = new MutableBlueRow(_table.schema);
+                var val = doc[n].ToString();
+                nr.setValue("id", id);
+                nr.setValue("doc_id", doc_id);
+                nr.setValue("key", key);
+                nr.setValue("value", val);
+                _table.addRow(nr);
+            }
+        }
+    }
+
+    [DataStreamUserRespresentation("mongodb_flat")]
+    public class MongoDBFlatStream : ViewStream {
+        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+        :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+
+        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+            mySetRealSource(rs);
+        }
+    }
+
+
+
+    [DataStreamUserRespresentation("newsreplay")]
+    public class NewsReplayStream : ViewStream {
+        public NewsReplayStream(string connection_string, string db_name, string collection_name,
+                                int buffer_sz, string fire_period) :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+
+        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+                                int buffer_sz, TimeSpan fire_period) {
+            ViewStream r;
+
+            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            r = new KeyValueToRelationalStream(r,
+                                                "doc_id",
+                                                "key",
+                                                "value");
+            r = new DateStandardizationStream(r,
+                                               "created_at",
+                                               "ISO8601");
+            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+
+            mySetRealSource(r);
+        }
+    }
+
+
+}

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -0,0 +1,172 @@
+namespace bts {
+   using BCG;
+
+   using ProtoBuf;
+
+   using System;
+   using System.Collections;
+   using System.Collections.Generic;
+
+   public class ProjectData {
+      // schema from the base stream
+      public readonly IBlueTableSchema in_schema;
+      // this is the projected schema
+      public readonly IBlueTableSchema out_schema;
+      // projected cols to the input cols, for cols who's names are changed
+      public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
+      public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
+                  IDictionary<string, string> mapped_to_orig_col_map_) {
+         in_schema = in_schema_;
+         out_schema = out_schema_;
+         mapped_to_orig_col_map = mapped_to_orig_col_map_;
+      }
+   }
+
+   public class NewProjectRow : IBlueRow {
+      #region IBlueRow members
+      private ProjectData _project_data;
+      private IBlueRow _row;
+      public NewProjectRow(ProjectData project_data, IBlueRow row) {
+         _project_data = project_data;
+         _row = row;
+      }
+      public IBlueTableSchema schema { get { return _project_data.out_schema; } }
+      #endregion
+
+      #region ICollection members
+      public int Count { get { return schema.columnNames.Length; } }
+
+      public bool IsSynchronized { get { return false; } }
+
+      public object SyncRoot { get { return this; } }
+
+      public void CopyTo(Array array, int index) {
+         int i = index;
+         foreach (object o in this as IEnumerable)
+            array.SetValue(o, i++);
+      }
+      #endregion
+
+      #region IDictionary members
+      public bool IsFixedSize { get { return true; } }
+
+      public bool IsReadOnly { get { return true; } }
+
+      public object this[object key] {
+         get {
+            if (key is string)
+               return this[(string)key];
+            return this[(int)key];
+         }
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+      }
+
+      public object this[int index] {
+         get {
+            return this[schema.columnInfo[index].colName];
+         }
+      }
+
+      public object this[string col_name] {
+         get {
+            string mapped_to_col = null;
+            if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
+               return _row[mapped_to_col];
+            }
+            return _row[col_name];
+         }
+      }
+
+      public ICollection Keys { get { return schema.columnNames; } }
+
+      public ICollection Values {
+         get {
+            object[] retVal = new object[schema.columnNames.Length];
+            CopyTo(retVal, 0);
+            return retVal;
+         }
+      }
+
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+      public bool Contains(object key) { return schema[(string)key] != null; }
+
+      /// <summary>
+      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
+      /// iterate through the values, typecast this into an IEnumerable
+      /// </summary>
+      /// <returns></returns>
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      #endregion
+
+   }
+
+   public class NewProjectStream : ViewStream, IStreamMapper {
+
+      private ViewStream _stream;
+      private string _project_expression;
+      private ProjectData _project_data = null;
+      //public ProjectData 
+      public NewProjectStream(ViewStream vs, string project_expression) {
+         _stream = vs;
+         _project_expression = project_expression;
+         _stream = new MapStream(vs, this);
+         mySetRealSource(_stream);
+      }
+
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+         _project_data = _create_project_data(input_schema, _project_expression);
+         return _project_data.out_schema;
+      }
+
+      public IBlueRow map(IBlueRow row) {
+         return new NewProjectRow(_project_data, row);
+      }
+
+      private static ProjectData _create_project_data(IBlueTableSchema input_schema,
+                                                      string project_expression) {
+         project_expression = project_expression.Trim();
+         string[] project_toks = project_expression.Split(' ');
+         HashSet<string> removed_cols = new HashSet<string>();
+         Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
+         Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
+         foreach (string p_tok in project_toks) {
+            if (p_tok.StartsWith("!")) {
+               string removed_col = p_tok.Substring(1);
+               removed_cols.Add(removed_col);
+            } else if (p_tok.Contains(":")) {
+               string[] map_toks = p_tok.Split(":");
+               string orig_col = map_toks[0];
+               string mapped_col = map_toks[1];
+               mapped_to_orig_map.Add(mapped_col, orig_col);
+               orig_to_mapped.Add(orig_col, mapped_col);
+            } else {
+               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+            }
+         }
+         // make the new schema
+         List<ColumnInfo> new_cols = new List<ColumnInfo>();
+         foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (removed_cols.Contains(ci.colName)) {
+               continue;
+            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
+               string mapped_name = orig_to_mapped[ci.colName];
+               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            } else {
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+            }
+         }
+         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
+         return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
+      }
+
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs

@@ -8,8 +8,8 @@ namespace BCG {
    using System.Collections.Generic;
    using System.Linq;
 
-   public class ProjectStream : ViewStream
-   {
+   [DataStreamUserRespresentation("oldproject")]
+   public class ProjectStream : ViewStream {
       private ProjectHelper myHelper;
       private string myColExp, myTableName;
       private ViewStream myVs;
@@ -19,8 +19,7 @@ namespace BCG {
       /// </summary>
       /// <param name="vs">stream to susbsribe to as the input stream</param>
       /// <param name="colExp">expression defining the column mapping</param>
-      public ProjectStream (ViewStream vs, string colExp)
-      {
+      public ProjectStream(ViewStream vs, string colExp) {
          myVs = vs;
          myColExp = colExp;
       }
@@ -30,12 +29,10 @@ namespace BCG {
       /// <param name="vs">stream to subscribe to as the input stream</param>
       /// <param name="colExp">expression defining the column mapping</param>
       /// <param name="tableName">name of the mapped table</param>
-      public ProjectStream (ViewStream vs, string colExp, string tableName)
-      {
+      public ProjectStream(ViewStream vs, string colExp, string tableName) {
          if (colExp == "!")
-            mySetRealSource (new ChangeTableNameStream (vs, tableName));
-         else
-         {
+            mySetRealSource(new ChangeTableNameStream(vs, tableName));
+         else {
             myVs = vs;
             myColExp = colExp;
             myTableName = tableName;
@@ -46,59 +43,51 @@ namespace BCG {
       /// </summary>
       /// <param name="vs">stream to subscribe to as the input stream</param>
       /// <param name="dic">dictionary mapping the aliased to the original</param>
-      public ProjectStream (ViewStream vs, IDictionary dic)
-         : this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) {}
+      public ProjectStream(ViewStream vs, IDictionary dic)
+         : this(vs, GenericDictionaryWrapper.Create<string, string>(dic)) { }
 
-      public ProjectStream (ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic)
-      {
+      public ProjectStream(ViewStream vs, IEnumerable<KeyValuePair<string, string>> dic) {
          myVs = vs;
          myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
       }
 
-      protected override void myOnFirstSubscriber()
-      {
-         myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
+      protected override void myOnFirstSubscriber() {
+         myVs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChange);
       }
 
-      protected override void myOnLastSubscriber()
-      {
-         myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
+      protected override void myOnLastSubscriber() {
+         myVs.onChange -= new BCG.DataStream.ChangeDelegate(myHandleChange); ;
       }
 
-      internal void myHandleChange (object newValue)
-      {
+      internal void myHandleChange(object newValue) {
          DataPacket dp = newValue as DataPacket;
-         if (myHelper != null)
-         {
+         if (myHelper != null) {
             _timer.start();
             IBlueRowCollection dels, adds;
-            BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
+            BlueDataUtility.getTrueDeletesAndAdds(dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
             _timer.stop();
             if (dels.Count + adds.Count > 0)
-               myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
-                                            new ProjectedTable (adds, myHelper),
-                                             new ProjectedTable (dels, myHelper)
+               myFireEvent(new DataPacket(new ProjectedTable(dp.newRows, myHelper),
+                                            new ProjectedTable(adds, myHelper),
+                                             new ProjectedTable(dels, myHelper)
                                           )
-                           );              
-         }
-         else
-         {  //first time a callback is received
+                           );
+         } else {  //first time a callback is received
             myHelper = myDic == null ?
-               new ProjectHelper (dp.schema, myColExp, myTableName) :
-               new ProjectHelper (dp.schema, myDic, myTableName);
+               new ProjectHelper(dp.schema, myColExp, myTableName) :
+               new ProjectHelper(dp.schema, myDic, myTableName);
 
             IBlueRowCollection temp;
-            myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
+            myFireEvent(new DataPacket(temp = new ProjectedTable(dp.newRows, myHelper),
                temp,
-               new BlueTable (temp.schema)
+               new BlueTable(temp.schema)
                )
                );
 
          }
       }
-      public void myHandleChangeTest(object newValue)
-      {
-          myHandleChange(newValue);
+      public void myHandleChangeTest(object newValue) {
+         myHandleChange(newValue);
       }
 
    }

src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs

@@ -3,24 +3,23 @@ using System.Collections.Generic;
 using System.Text;
 
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
    /// </summary>
-   public enum DateConversion
-   {
+   public enum DateConversion {
       ConvertDoubleToDateTime = 0,
       ChopTime,
       UTCToCurrent,
       ConvertStringToDateTime,
       ConvertDateTimeToMilliseconds,
-      SecondsOfDay
+      SecondsOfDay,
+      ISO8601,
+      EpochToDatetime
    }
 
    [DataStreamUserRespresentation("dateStandardize")]
-   public class DateStandardizationStream : ViewStream, IStreamMapper
-   {
+   public class DateStandardizationStream : ViewStream, IStreamMapper {
       private IBlueTableSchema mySchema;
       private DateConversion myConversion;
       private string myConversionColName;
@@ -31,12 +30,13 @@ namespace BCG
          {DateConversion.UTCToCurrent                 , typeof(DateTime)},
          {DateConversion.ConvertStringToDateTime      , typeof(DateTime)},
          {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
-         {DateConversion.SecondsOfDay                 , typeof(Double)}
+         {DateConversion.SecondsOfDay                 , typeof(Double)},
+         {DateConversion.ISO8601                      , typeof(DateTime)},
+         {DateConversion.EpochToDatetime              , typeof(DateTime)}
       };
 
 
-      private static DateConversion _parse(string s)
-      {
+      private static DateConversion _parse(string s) {
          DateConversion rv = DateConversion.ChopTime;
          Enum.TryParse<DateConversion>(s, false, out rv);
          return rv;
@@ -45,11 +45,9 @@ namespace BCG
       /// for DSQA
       /// </summary>
       public DateStandardizationStream(ViewStream vs, string colName, string conversion)
-         : this(vs, colName, _parse(conversion))
-      { }
+         : this(vs, colName, _parse(conversion)) { }
 
-      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
-      {
+      public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) {
          myConversionColName = colName;
          myConversion = conversion;
          mySetRealSource(new MapStream(vs, this));
@@ -57,24 +55,18 @@ namespace BCG
 
       #region IStreamMapper Members
 
-      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
-      {
-         if (mySchema == null)
-         {
+      public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+         if (mySchema == null) {
             List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (ColumnInfo c in inputSchema.columnInfo)
-            {
-               if (c.colName == myConversionColName)
-               {
+            foreach (ColumnInfo c in inputSchema.columnInfo) {
+               if (c.colName == myConversionColName) {
                   Type convType;
-                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
-                  {
+                  if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) {
                      //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
                      convType = typeof(DateTime);
                   }
                   cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
-               }
-               else
+               } else
                   cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
             }
             mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
@@ -86,56 +78,59 @@ namespace BCG
 
       #region IDynamicStreamMapper Members
 
-      public IBlueRow map(IBlueRow row)
-      {
-         return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
+      public IBlueRow map(IBlueRow row) {
+         MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
+         rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
+         return rv;
+         //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
       }
 
       #endregion
 
-      public static object mapDateTime(object objIn, DateConversion conv)
-      {
-         if (objIn == DBNull.Value || objIn == null)
-         {
+      public static object mapDateTime(object obj_in, DateConversion conv) {
+         if (obj_in == DBNull.Value || obj_in == null) {
             return DBNull.Value;
-         }
-         else
-         {
-            if (conv == DateConversion.ChopTime)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.Date;
-            }
-            else if (conv == DateConversion.ConvertDoubleToDateTime)
-            {
-               double dtIn = (double)objIn;
-               return DateTime.FromOADate(dtIn);
-            }
-            else if (conv == DateConversion.UTCToCurrent)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.ToLocalTime();
-            }
-            else if (conv == DateConversion.ConvertStringToDateTime)
-            {
-               return DateTime.Parse((string)objIn);
-            }
-            else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               return dtIn.TimeOfDay.TotalMilliseconds;
-            }
-            else if (conv == DateConversion.SecondsOfDay)
-            {
-               DateTime dtIn = (DateTime)objIn;
-               DateTime start_of_day = dtIn.Date;
-               TimeSpan day_period = dtIn - start_of_day;
-               return day_period.TotalSeconds;
-            }
-            else
-            {
-               bts.Log.Gen.info("incorrect type");
-               return null;
+         } else {
+            switch (conv) {
+               case DateConversion.ChopTime: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     return dtIn.Date;
+                  }
+               case DateConversion.ConvertDoubleToDateTime: {
+                     double dtIn = (double)obj_in;
+                     return DateTime.FromOADate(dtIn);
+                  }
+               case DateConversion.UTCToCurrent: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     return dtIn.ToLocalTime();
+                  }
+               case DateConversion.ConvertStringToDateTime: {
+                     return DateTime.Parse((string)obj_in);
+                  }
+               case DateConversion.ConvertDateTimeToMilliseconds: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     return dtIn.TimeOfDay.TotalMilliseconds;
+                  }
+               case DateConversion.SecondsOfDay: {
+                     DateTime dtIn = (DateTime)obj_in;
+                     DateTime start_of_day = dtIn.Date;
+                     TimeSpan day_period = dtIn - start_of_day;
+                     return day_period.TotalSeconds;
+                  }
+               case DateConversion.ISO8601: {
+                     string iso8601Date = obj_in.ToString();
+                     return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind);
+                  }
+               case DateConversion.EpochToDatetime: {
+                     double d = Convert.ToDouble(obj_in);
+                     long unix_time_seconds = (long)d;
+                     DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+                     return epoch.AddSeconds(unix_time_seconds);
+                  }
+               default: {
+                     bts.Log.Gen.info("incorrect type");
+                     return DBNull.Value;
+                  }
             }
          }
       }

src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs

@@ -8,38 +8,36 @@ namespace BCG {
       private ViewStream mySrc;
       private IBlueTableSchema mySchema = null;
 
-      public AlphabetizeColumnsStream (ViewStream src) {
+      public AlphabetizeColumnsStream(ViewStream src) {
          mySrc = src;
       }
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          mySrc.onChange += new ChangeDelegate(mySrc_onChange);
       }
-      protected override void myOnLastSubscriber () 
-      {
-         mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
+      protected override void myOnLastSubscriber() {
+         mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
       }
-      private void mySrc_onChange (object newValue) {
+      private void mySrc_onChange(object newValue) {
          DataPacket dp = (DataPacket)newValue;
          if (mySchema == null) {
-            SortedList alphabetical = new SortedList ();
+            SortedList alphabetical = new SortedList();
             foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
-               alphabetical.Add (cinfo.colName,cinfo);
+               alphabetical.Add(cinfo.colName, cinfo);
             ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
-            alphabetical.Values.CopyTo (cinfos,0);
+            alphabetical.Values.CopyTo(cinfos, 0);
             for (int i = 0; i < cinfos.Length; ++i)
-               cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue);
-            mySchema = new BlueTableSchema (cinfos,dp.schema.tableName);
+               cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType);
+            mySchema = new BlueTableSchema(cinfos, dp.schema.tableName);
          }
          BlueTable
-            deleted = new BlueTable (mySchema),
-            added = new BlueTable (mySchema),
-            newrows = new BlueTable (mySchema);
-         deleted.addRows (dp.deletedRows);
-         added.addRows (dp.addedRows);
-         newrows.addRows (dp.newRows);
-      
-         myFireEvent (new DataPacket (newrows,added,deleted));
+            deleted = new BlueTable(mySchema),
+            added = new BlueTable(mySchema),
+            newrows = new BlueTable(mySchema);
+         deleted.addRows(dp.deletedRows);
+         added.addRows(dp.addedRows);
+         newrows.addRows(dp.newRows);
+
+         myFireEvent(new DataPacket(newrows, added, deleted));
       }
    }
 }

src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs

@@ -1,71 +1,61 @@
 using System;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Publishes the rows in the form of a circular buffer, ie if the buffer size is n
    /// then the n'th row being puclished results in the deletion of the 0th row
    /// </summary>
-	public class CircularPublisher
-	{
+	public class CircularPublisher {
       private CircularRowCollection myBuffer;
       private PubTableBroker myBroker;
       private int myBufferSize;
-		public CircularPublisher (int bufferSize)
-		{
+      public CircularPublisher(int bufferSize) {
          myBufferSize = bufferSize;
-		}
-
-      public CircularPublisher (int bufferSize, IBlueRowCollection rows) 
-         : this (bufferSize)
-      {
-         publishRows (rows);
       }
 
-      public void publishRow (IBlueRow nr)
-      {
-         if (myBuffer == null)
-         {           
-            myBuffer = new CircularRowCollection (
-               new BlueTableSchema (nr.schema.tableName, nr.schema.columnInfo), 
-               myBufferSize);
-            myBroker = new PubTableBroker ();
-            myBroker.createTable (myBuffer);
-         }
-         IBlueRow br = myBuffer.addRowWithReturn (nr);
-         myBroker.makeBatchChanges (myBuffer, 
-            myReturnCollection (br),
-            myReturnCollection (nr));
+      public CircularPublisher(int bufferSize, IBlueRowCollection rows)
+         : this(bufferSize) {
+         publishRows(rows);
       }
 
-      public void publishRows (IBlueRowCollection nrs)
-      {
-         if (myBuffer == null)
-         {           
-            myBuffer = new CircularRowCollection (
-               new BlueTableSchema (BlueDataUtility.extractRow (nrs, 0).schema.tableName, 
-               BlueDataUtility.extractRow (nrs, 0).schema.columnInfo),
+      public void publishRow(IBlueRow nr) {
+         if (myBuffer == null) {
+            myBuffer = new CircularRowCollection(
+               new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
                myBufferSize);
-            myBroker = new PubTableBroker ();
-            myBroker.createTable (myBuffer);
+            myBroker = new PubTableBroker();
+            myBroker.create_table(myBuffer);
          }
-         BlueTable deletes = new BlueTable (myBuffer.schema);
-         foreach (IBlueRow nr in nrs)
-         {
-            IBlueRow dr = myBuffer.addRowWithReturn (nr);
+         IBlueRow br = myBuffer.addRowWithReturn(nr);
+         myBroker.make_batch_changes(myBuffer,
+            myReturnCollection(br),
+            myReturnCollection(nr));
+      }
+
+      public void publishRows(IBlueRowCollection nrs) {
+         if (myBuffer == null) {
+            myBuffer = new CircularRowCollection(
+               new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
+               BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
+               myBufferSize);
+            myBroker = new PubTableBroker();
+            myBroker.create_table(myBuffer);
+         }
+         BlueTable deletes = new BlueTable(myBuffer.schema);
+         foreach (IBlueRow nr in nrs) {
+            IBlueRow dr = myBuffer.addRowWithReturn(nr);
             if (dr != null)
-               deletes.addRow (dr);
+               deletes.addRow(dr);
          }
-         myBroker.makeBatchChanges (myBuffer, deletes, nrs);
+         myBroker.make_batch_changes(myBuffer, deletes, nrs);
       }
 
-      private IBlueRowCollection myReturnCollection (IBlueRow r)
-      {
+      private IBlueRowCollection myReturnCollection(IBlueRow r) {
          if (r == null)
-            return new BlueTable (myBuffer.schema);
-         BlueTable retVal = new BlueTable (r.schema);
-         retVal.addRow (r);
+            return new BlueTable(myBuffer.schema);
+         BlueTable retVal = new BlueTable(r.schema);
+         retVal.addRow(r);
          return retVal;
       }
-	}
+   }
 }

src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs

@@ -1,184 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq.Expressions;
-using System.Xml.Serialization;
-
-using BCG.Utilities;
-
-namespace BCG {
-   /// <summary>
-   /// This class will be used by the clients to publish tabular data to the pubsub
-   /// </summary>
-   public class PubTableBroker {
-      protected PublisherCore _publisher_core;
-      protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
-      protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
-
-      protected bool myWaitingForFirstOp = false;
-
-      public PubTableBroker()
-         : this(new PublisherCore()) { }
-
-      public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
-
-      public PubTableBroker(IBlueRowCollection[] tablesToPublish)
-         : this(new PublisherCore(), tablesToPublish) { }
-
-      public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
-         _publisher_core = publisher;
-
-         _publisher_core.beginTransaction();
-         foreach (IBlueRowCollection table in tablesToPublish)
-            createTable(table);
-         _publisher_core.commitTransaction();
-      }
-
-      public void beginTransaction() {
-         myWaitingForFirstOp = true;
-      }
-
-      public void commitTransaction() {
-         bts.Log.Gen.info("PubTableBroker.commitTransaction");
-         if (myWaitingForFirstOp)
-            myWaitingForFirstOp = false;
-         else
-            _publisher_core.commitTransaction();
-      }
-
-      public void rollbackTransaction() {
-         bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
-         if (myWaitingForFirstOp)
-            myWaitingForFirstOp = false;
-         else
-            _publisher_core.abortTransaction();
-      }
-
-      public virtual void createTable(IBlueRowCollection newTable) {
-         string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
-         foreach (IBlueRow r in newTable) {
-            log_msg += $"id: {r["id"]}";
-         }
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(newTable.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = newTable.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
-            table_publisher = _create_pubtable_publisher(name_key);
-            _publisher_map.Add(name_key, table_publisher);
-            _schema_map.Add(name_key, newTable.schema);
-         }
-         table_publisher.createTable(newTable);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void addRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = r.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Row can not be added, as the table that has not been created yet");
-         else
-            table_publisher.addRow(r);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void deleteRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = r.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Row can not be removed, as the table has not been created yet");
-         else
-            table_publisher.deleteRow(r);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void changeRow(IBlueRow r) {
-         string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(r.schema);
-         myCheckTransaction();
-         IPubTablePublisher table_publisher = null;
-         var name_key = r.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Row can not be changed, as the table has not been created yet");
-         else
-            table_publisher.changeRow(r);
-      }
-
-      /// <summary>
-      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
-      /// will be thrown
-      /// </summary>
-      /// <param name="r"></param>
-      public virtual void makeBatchChanges(IBlueRowCollection newRows,
-                                             IBlueRowCollection deletedRows,
-                                             IBlueRowCollection addedRows) {
-         string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
-         foreach (IBlueRow r in deletedRows)
-            log_msg += $"id: {r["id"]} ";
-         log_msg += "addedRows: ";
-         foreach (IBlueRow r in addedRows)
-            log_msg += $"id: {r["id"]}";
-         bts.Log.Gen.info(log_msg);
-         _check_schema_consistency(newRows.schema);
-         _check_schema_consistency(deletedRows.schema);
-         _check_schema_consistency(addedRows.schema);
-         IPubTablePublisher table_publisher = null;
-         var name_key = newRows.schema.tableName.ToLower();
-         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
-            throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
-         else
-            table_publisher.batchRemoveAdd(deletedRows, addedRows);
-      }
-
-      public virtual void deleteTableRows(string tableName) {
-         myCheckTransaction();
-         (_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
-            new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
-      }
-
-      public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
-
-      public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
-
-      protected void myCheckTransaction() {
-         if (myWaitingForFirstOp)
-            _publisher_core.beginTransaction();
-         myWaitingForFirstOp = false;
-      }
-
-      protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
-         bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
-         return new PubTablePublisher(tag, _publisher_core);
-      }
-
-      private void _check_schema_consistency(IBlueTableSchema schema_arg) {
-         IBlueTableSchema schema;
-         if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
-            bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -0,0 +1,31 @@
+namespace BCG {
+   // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
+   // and hence epublishes it out to the PS
+   public class PublisherStream : ViewStream {
+      private readonly ViewStream _main_stream;
+      private readonly string _name;
+      private ViewPublisher _vs_publisher;
+      private readonly PubTableBroker _broker = new PubTableBroker();
+
+      public PublisherStream(ViewStream main_stream, string name) {
+         _main_stream = main_stream;
+         _name = name;
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vs_publisher = new ViewPublisher(_name, _broker, _main_stream);
+         _main_stream.onChange += new DataStream.ChangeDelegate(_handle_update);
+      }
+
+      protected override void myOnLastSubscriber() {
+         _main_stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
+         _vs_publisher.Dispose();
+      }
+
+      private void _handle_update(object o) {
+         myFireEvent(o);
+      }
+
+
+   }
+}

src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs

@@ -7,48 +7,32 @@ using BCG.Utilities;
 namespace BCG {
    public class ViewPublisher : IDisposable {
       protected ViewPublisher() { }
+      private readonly PubTableBroker _broker;
+      protected readonly ViewStream _stream;
+      private int _n_updates = 0;
 
       /// If an name IsNullOrEmpty then it uses the name
       /// in the schema (be careful with that because this
       /// could lead to double publishes if the schema name
       /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
-         myName = name;
-         myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
-         myBroker = br;
-         myFirstTime = true;
-         myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+      public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
+         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
+         _broker = broker;
+         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
       }
 
       public void Dispose() {
-         myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
-         if (!String.IsNullOrEmpty(myName))
-            myBroker.clearTable(myName);
+         _stream.onChange -= new DataStream.ChangeDelegate(_handle_update);
       }
 
-      private void myChangeHandler(object o) {
-         DataPacket dp = (DataPacket)o;
-         if (myFirstTime) {
-            myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
-            myLastRowCount = dp.newRows.Count;
-            myBroker.createTable(dp.newRows);
-            myFirstTime = false;
+      private void _handle_update(object o) {
+         DataPacket dp = o as DataPacket;
+         if (_n_updates == 0) {
+            _broker.create_table(dp.newRows);
+            _n_updates++;
          } else {
-            if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
-               myLastRowCount = dp.newRows.Count;
-               myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
-            }
+            _broker.make_batch_changes(dp.newRows, dp.deletedRows, dp.addedRows);
          }
       }
-
-      public int getLastRowPublishedCount() { return myLastRowCount; }
-
-      public bool hasPublished() { return !myFirstTime; }
-
-      private int myLastRowCount = -1;
-      private string myName;
-      private PubTableBroker myBroker;
-      protected ViewStream myView;
-      private bool myFirstTime;
    }
 }

src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs

@@ -1,92 +0,0 @@
-using System;
-using BCG.Utilities;
-
-namespace BCG
-{
-	public class ViewStreamPublisher
-	{
-      private IDynamicStreamMapper myMapper;
-      private PubTableBroker myBroker;
-      private IBlueRowCollection myOutputTable;
-      private IBlueRowCollection myInputTable;
-
-      public ViewStreamPublisher (PubTableBroker broker)
-      {
-         myBroker = broker;
-      }
-
-      public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper)
-         : this (broker)
-      {
-         myMapper = mapper;
-      }
-
-		public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs)
-         : this (broker, mapper)
-		{
-         vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges);
-      }
-
-      private void myHandleChanges (object newValue)
-      {
-         DataPacket dp = newValue as DataPacket;
-         if (myMapper == null)
-         {
-            if (myOutputTable == null)
-            {
-               myBroker.createTable (dp.newRows);
-            }
-            else
-            {
-               myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows);
-            }
-            myOutputTable = dp.newRows;
-         }
-         else
-         {
-            if (myOutputTable != null)
-            {
-               BlueTable dRows = new BlueTable (myOutputTable.schema),
-                  aRows = new BlueTable (myOutputTable.schema);
-               foreach (IBlueRow dRow in dp.deletedRows)
-               {
-                  (myOutputTable as BlueTable).deleteRow ((int)dRow["id"]);
-                  dRows.addRow (myMapper.map (dRow));
-               }
-               foreach (IBlueRow aRow in dp.addedRows)
-               {
-                  IBlueRow r = myMapper.map (aRow);
-                  (myOutputTable as BlueTable).addRow (r);
-                  aRows.addRow (r);
-               }
-               myBroker.makeBatchChanges (myOutputTable, dRows, aRows);
-            }
-            else
-            {
-               if (dp.newRows.Count > 0)
-               {
-                  myOutputTable = new BlueTable (
-                  myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema);
-                  myHandleChanges (newValue);
-               }
-            }
-         }
-      }
-
-      public void addRows (IBlueRowCollection aRows)
-      {
-         if (myInputTable != null)
-            (myInputTable as BlueTable).addRows (aRows);
-         else
-            myInputTable = new BlueTable (aRows);
-         myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema)));
-      }
-
-      public void deleteRows (IBlueRowCollection dRows)
-      {
-         // BlueEventLog.logAssert (myInputTable != null);
-         (myInputTable as BlueTable).deleteRows (dRows);
-         myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows));
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs

@@ -1,45 +1,42 @@
 using System;
 
-namespace BCG
-{
-	/// <summary>
-	/// Allowed types
-	/// </summary>
-	public class BlueDataValidTypes
-	{
-      public static readonly Type BOOL = typeof(bool);
-      public static readonly Type BYTE = typeof(byte);
-      public static readonly Type CHAR = typeof(char);
-      public static readonly Type DECIMAL = typeof(decimal);
-      public static readonly Type DOUBLE = typeof(double);
-      public static readonly Type SHORT = typeof(short);
-      public static readonly Type INT = typeof(int);
-      public static readonly Type LONG = typeof(long);
-      public static readonly Type SBYTE = typeof(sbyte);
-      //public static readonly Type FLOAT = typeof(float);
-      public static readonly Type STRING = typeof(string);
-      //public static readonly Type USHORT = typeof(ushort);
-      public static readonly Type UINT = typeof(uint);
-      public static readonly Type ULONG = typeof(ulong);
-      public static readonly Type DATETIME = typeof(DateTime);
-      public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
-      public static readonly Type SINGLE  = typeof(Single);
-      public static readonly Type BYTEARRAY = typeof(byte[]);
+namespace BCG {
+    /// <summary>
+    /// Allowed types
+    /// </summary>
+    public class BlueDataValidTypes {
+        public static readonly Type BOOL = typeof(bool);
+        public static readonly Type BYTE = typeof(byte);
+        public static readonly Type CHAR = typeof(char);
+        public static readonly Type DECIMAL = typeof(decimal);
+        public static readonly Type DOUBLE = typeof(double);
+        public static readonly Type SHORT = typeof(short);
+        public static readonly Type INT = typeof(int);
+        public static readonly Type LONG = typeof(long);
+        public static readonly Type SBYTE = typeof(sbyte);
+        //public static readonly Type FLOAT = typeof(float);
+        public static readonly Type STRING = typeof(string);
+        //public static readonly Type USHORT = typeof(ushort);
+        public static readonly Type UINT = typeof(uint);
+        public static readonly Type ULONG = typeof(ulong);
+        public static readonly Type DATETIME = typeof(DateTime);
+        public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
+        public static readonly Type SINGLE = typeof(Single);
+        public static readonly Type BYTEARRAY = typeof(byte[]);
 
-      public static int getValueTypeSize (Type t)
-      {
-          if (t == BOOL || t == BYTE  || t == SBYTE)
-              return 1;
-          else if (t == SHORT || t == CHAR)
-              return 2;
-          else if (t == INT || t == UINT || t == SINGLE)
-              return 4;
-          else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
-              return 8;
-          else if (t == DECIMAL)
-              return 16;
-          else
-              throw new ApplicationException("Unsupported Data Type");
-      }
-	}
+        public static int getValueTypeSize(Type t) {
+            if (t == BOOL || t == BYTE || t == SBYTE)
+                return 1;
+            else if (t == SHORT || t == CHAR)
+                return 2;
+            else if (t == INT || t == UINT || t == SINGLE)
+                return 4;
+            else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
+                return 8;
+            else if (t == DECIMAL)
+                return 16;
+            else
+                throw new ApplicationException("Unsupported Data Type");
+        }
+    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs

@@ -1,116 +1,117 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-/*
- * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. 
- * if groupNo is zero, then the group is not stord in a lookup table at all. 
- * 
- * 
- * 
- */
-namespace BCG
-{
-   public class BlueRowLookupGroup
-   {
-      BlueTableSchema myEntrySchema;
-      public IBlueTableSchema entrySchema
-      {
-         get { return myEntrySchema; }
-         //cant' set this value, have to build it from the row's schema
+//TODORemove
+// /*
+//  * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. 
+//  * if groupNo is zero, then the group is not stord in a lookup table at all. 
+//  * 
+//  * 
+//  * 
+//  */
+// namespace BCG
+// {
+//    public class BlueRowLookupGroup
+//    {
+//       BlueTableSchema myEntrySchema;
+//       public IBlueTableSchema entrySchema
+//       {
+//          get { return myEntrySchema; }
+//          //cant' set this value, have to build it from the row's schema
 
-      }
+//       }
 
-      //for faster access and such
-      private int[] myGroupColNos;
-      public int[] groupColNos
-      {
-         get { return myGroupColNos; }
+//       //for faster access and such
+//       private int[] myGroupColNos;
+//       public int[] groupColNos
+//       {
+//          get { return myGroupColNos; }
 
-      }
-      private string[] myGroupColNames;
-      public string[] groupColNames
-      {
-         get { return myGroupColNames; }
-      }
+//       }
+//       private string[] myGroupColNames;
+//       public string[] groupColNames
+//       {
+//          get { return myGroupColNames; }
+//       }
 
-      private short myGroupNumber;
-      public short groupNumber
-      {
-         get { return myGroupNumber; }
+//       private short myGroupNumber;
+//       public short groupNumber
+//       {
+//          get { return myGroupNumber; }
 
-      }
-      
-      /* how many bits are the ids into the table?
-       * In c# everything is stored as integers, but when it goes across the wire
-       * the data can be represented as a byte, a short, or an int 
-       */
-      private short myTableSize;
-      public short tableSize
-      {
-         get { return myTableSize; }
+//       }
 
-      }
+//       /* how many bits are the ids into the table?
+//        * In c# everything is stored as integers, but when it goes across the wire
+//        * the data can be represented as a byte, a short, or an int 
+//        */
+//       private short myTableSize;
+//       public short tableSize
+//       {
+//          get { return myTableSize; }
 
-      //if the group number is 0, the group is not stored in tables for serialization
-      public bool isStoredInTable
-      {
-         get { return myGroupNumber != 0; }
+//       }
 
-      }
-      public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
-      {
+//       //if the group number is 0, the group is not stored in tables for serialization
+//       public bool isStoredInTable
+//       {
+//          get { return myGroupNumber != 0; }
 
-         myGroupNumber = num;
-         myTableSize = sizeInBits;
-         myGroupColNames = cols ;
-      }
+//       }
+//       public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
+//       {
 
-      /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
-       each of these entries in the lookup table.  This function should be called by the LookupTableSchema
-       during its constructor. 
-       
-       This function also caches the column numbers for this group, for quicker access*/
-      public void buildEntrySchema(LookupTableSchema theSchema)
-      {
-         ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
-         myGroupColNos = new int[myGroupColNames.Length];
-         //the first column in our table is always the ID
-         int idColumn = 0;
-         columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
-         //go through each of the columns
-         for (int col = 1; col < columns.Length; ++col)
-         {
-            ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
-            if (columnInInputTable.isConstant)
-               columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
-            else
-               columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
+//          myGroupNumber = num;
+//          myTableSize = sizeInBits;
+//          myGroupColNames = cols ;
+//       }
 
-            myGroupColNos[col - 1] = columnInInputTable.colNo;
-         }
+//       /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
+//        each of these entries in the lookup table.  This function should be called by the LookupTableSchema
+//        during its constructor. 
 
-         
+//        This function also caches the column numbers for this group, for quicker access*/
+//       public void buildEntrySchema(LookupTableSchema theSchema)
+//       {
+//          ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
+//          myGroupColNos = new int[myGroupColNames.Length];
+//          //the first column in our table is always the ID
+//          int idColumn = 0;
+//          columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
+//          //go through each of the columns
+//          for (int col = 1; col < columns.Length; ++col)
+//          {
+//             ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
+//             if (columnInInputTable.isConstant)
+//                columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
+//             else
+//                columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
 
-         myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
-
-      }
-
-      /*takes a row as input, and extracts the members of the group from the row */
-      public object[] extractGroupFromRow(IBlueRow input)
-      {
-         int numCols = myGroupColNames.Length;
-         object[] results = new object[numCols];
-
-         for (int col = 0; col < numCols; ++col)
-            results[col] = input[myGroupColNos[col]];
-         
-         return results;
-
-      }
+//             myGroupColNos[col - 1] = columnInInputTable.colNo;
+//          }
 
 
 
-   }
-}
+//          myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
+
+//       }
+
+//       /*takes a row as input, and extracts the members of the group from the row */
+//       public object[] extractGroupFromRow(IBlueRow input)
+//       {
+//          int numCols = myGroupColNames.Length;
+//          object[] results = new object[numCols];
+
+//          for (int col = 0; col < numCols; ++col)
+//             results[col] = input[myGroupColNos[col]];
+
+//          return results;
+
+//       }
+
+
+
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs

@@ -8,8 +8,7 @@ using System.Linq;
 
 using BCG.Utilities.General;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
    /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
@@ -17,112 +16,105 @@ namespace BCG
    /// the respective data, sorted according to the ColumnOrder.
    /// </summary>
    [Serializable]
-   public class BlueTableSchema : IBlueTableSchema
-   {
-      public class BlueTableSchemaSerializationHelper
-      {
-         public static object parse(Type t, string data)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return Boolean.Parse(data);
-            else if (t == BlueDataValidTypes.BYTE)
-               return Byte.Parse(data);
-            else if (t == BlueDataValidTypes.CHAR)
-               return char.Parse(data);
-            else if (t == BlueDataValidTypes.DATETIME)
-               return DateTime.Parse(data);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return Double.Parse(data);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return Decimal.Parse(data);
-            else if (t == BlueDataValidTypes.INT)
-               return int.Parse(data);
-            else if (t == BlueDataValidTypes.LONG)
-               return long.Parse(data);
-            else if (t == BlueDataValidTypes.SBYTE)
-               return SByte.Parse(data);
-            else if (t == BlueDataValidTypes.STRING)
-               return data;
-            else if (t == BlueDataValidTypes.UINT)
-               return uint.Parse(data);
-            else if (t == BlueDataValidTypes.ULONG)
-               return ulong.Parse(data);
-            else if (t == BlueDataValidTypes.SINGLE)
-               return Single.Parse(data);
-            else
-               throw new ApplicationException("Can not be parsed");
-         }
-         public static void serialize(BinaryWriter bw, Type t, object o)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               bw.Write((bool)o);
-            else if (t == BlueDataValidTypes.BYTE)
-               bw.Write((byte)o);
-            else if (t == BlueDataValidTypes.CHAR)
-               bw.Write((char)o);
-            else if (t == BlueDataValidTypes.DATETIME)
-               bw.Write(((DateTime)o).Ticks);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               bw.Write((double)o);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               bw.Write((decimal)o);
-            else if (t == BlueDataValidTypes.INT)
-               bw.Write((int)o);
-            else if (t == BlueDataValidTypes.LONG)
-               bw.Write((long)o);
-            else if (t == BlueDataValidTypes.SBYTE)
-               bw.Write((sbyte)o);
-            else if (t == BlueDataValidTypes.STRING)
-               bw.Write(o as string);
-            else if (t == BlueDataValidTypes.SINGLE)
-               bw.Write((Single)o);
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-            {
-               byte[] bs = o as byte[];
-               bw.Write(bs.Length);
-               bw.Write(bs);
-            }
-            else if (t == BlueDataValidTypes.UINT)
-               bw.Write((uint)o);
-            else if (t == BlueDataValidTypes.ULONG)
-               bw.Write((ulong)o);
-            else
-               throw new ApplicationException("Unsupported type");
-         }
+   public class BlueTableSchema : IBlueTableSchema {
+      public class BlueTableSchemaSerializationHelper {
+         // public static object parse(Type t, string data) {
+         //    if (t == BlueDataValidTypes.BOOL)
+         //       return Boolean.Parse(data);
+         //    else if (t == BlueDataValidTypes.BYTE)
+         //       return Byte.Parse(data);
+         //    else if (t == BlueDataValidTypes.CHAR)
+         //       return char.Parse(data);
+         //    else if (t == BlueDataValidTypes.DATETIME)
+         //       return DateTime.Parse(data);
+         //    else if (t == BlueDataValidTypes.DOUBLE)
+         //       return Double.Parse(data);
+         //    else if (t == BlueDataValidTypes.DECIMAL)
+         //       return Decimal.Parse(data);
+         //    else if (t == BlueDataValidTypes.INT)
+         //       return int.Parse(data);
+         //    else if (t == BlueDataValidTypes.LONG)
+         //       return long.Parse(data);
+         //    else if (t == BlueDataValidTypes.SBYTE)
+         //       return SByte.Parse(data);
+         //    else if (t == BlueDataValidTypes.STRING)
+         //       return data;
+         //    else if (t == BlueDataValidTypes.UINT)
+         //       return uint.Parse(data);
+         //    else if (t == BlueDataValidTypes.ULONG)
+         //       return ulong.Parse(data);
+         //    else if (t == BlueDataValidTypes.SINGLE)
+         //       return Single.Parse(data);
+         //    else
+         //       throw new ApplicationException("Can not be parsed");
+         // }
+         // public static void serialize(BinaryWriter bw, Type t, object o) {
+         //    if (t == BlueDataValidTypes.BOOL)
+         //       bw.Write((bool)o);
+         //    else if (t == BlueDataValidTypes.BYTE)
+         //       bw.Write((byte)o);
+         //    else if (t == BlueDataValidTypes.CHAR)
+         //       bw.Write((char)o);
+         //    else if (t == BlueDataValidTypes.DATETIME)
+         //       bw.Write(((DateTime)o).Ticks);
+         //    else if (t == BlueDataValidTypes.DOUBLE)
+         //       bw.Write((double)o);
+         //    else if (t == BlueDataValidTypes.DECIMAL)
+         //       bw.Write((decimal)o);
+         //    else if (t == BlueDataValidTypes.INT)
+         //       bw.Write((int)o);
+         //    else if (t == BlueDataValidTypes.LONG)
+         //       bw.Write((long)o);
+         //    else if (t == BlueDataValidTypes.SBYTE)
+         //       bw.Write((sbyte)o);
+         //    else if (t == BlueDataValidTypes.STRING)
+         //       bw.Write(o as string);
+         //    else if (t == BlueDataValidTypes.SINGLE)
+         //       bw.Write((Single)o);
+         //    else if (t == BlueDataValidTypes.BYTEARRAY) {
+         //       byte[] bs = o as byte[];
+         //       bw.Write(bs.Length);
+         //       bw.Write(bs);
+         //    } else if (t == BlueDataValidTypes.UINT)
+         //       bw.Write((uint)o);
+         //    else if (t == BlueDataValidTypes.ULONG)
+         //       bw.Write((ulong)o);
+         //    else
+         //       throw new ApplicationException("Unsupported type");
+         // }
 
-         public static object deserialize(BinaryReader br, Type t)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return br.ReadBoolean();
-            else if (t == BlueDataValidTypes.BYTE)
-               return br.ReadByte();
-            else if (t == BlueDataValidTypes.CHAR)
-               return br.ReadChar();
-            else if (t == BlueDataValidTypes.DATETIME)
-               return new DateTime(br.ReadInt64());
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return br.ReadDouble();
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return br.ReadDecimal();
-            else if (t == BlueDataValidTypes.INT)
-               return br.ReadInt32();
-            else if (t == BlueDataValidTypes.LONG)
-               return br.ReadInt64();
-            else if (t == BlueDataValidTypes.SBYTE)
-               return br.ReadSByte();
-            else if (t == BlueDataValidTypes.STRING)
-               return br.ReadString();
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-               return br.ReadBytes(br.ReadInt32());
-            else if (t == BlueDataValidTypes.UINT)
-               return br.ReadUInt32();
-            else if (t == BlueDataValidTypes.ULONG)
-               return br.ReadUInt64();
-            else if (t == BlueDataValidTypes.SINGLE)
-               return br.ReadSingle();
-            else
-               throw new ApplicationException("Unsupported type");
-         }
+         // public static object deserialize(BinaryReader br, Type t) {
+         //    if (t == BlueDataValidTypes.BOOL)
+         //       return br.ReadBoolean();
+         //    else if (t == BlueDataValidTypes.BYTE)
+         //       return br.ReadByte();
+         //    else if (t == BlueDataValidTypes.CHAR)
+         //       return br.ReadChar();
+         //    else if (t == BlueDataValidTypes.DATETIME)
+         //       return new DateTime(br.ReadInt64());
+         //    else if (t == BlueDataValidTypes.DOUBLE)
+         //       return br.ReadDouble();
+         //    else if (t == BlueDataValidTypes.DECIMAL)
+         //       return br.ReadDecimal();
+         //    else if (t == BlueDataValidTypes.INT)
+         //       return br.ReadInt32();
+         //    else if (t == BlueDataValidTypes.LONG)
+         //       return br.ReadInt64();
+         //    else if (t == BlueDataValidTypes.SBYTE)
+         //       return br.ReadSByte();
+         //    else if (t == BlueDataValidTypes.STRING)
+         //       return br.ReadString();
+         //    else if (t == BlueDataValidTypes.BYTEARRAY)
+         //       return br.ReadBytes(br.ReadInt32());
+         //    else if (t == BlueDataValidTypes.UINT)
+         //       return br.ReadUInt32();
+         //    else if (t == BlueDataValidTypes.ULONG)
+         //       return br.ReadUInt64();
+         //    else if (t == BlueDataValidTypes.SINGLE)
+         //       return br.ReadSingle();
+         //    else
+         //       throw new ApplicationException("Unsupported type");
+         // }
       }
 
       protected Hashtable myIndexByColName = new Hashtable();
@@ -132,29 +124,24 @@ namespace BCG
 
       public BlueTableSchema() { }
 
-      public BlueTableSchema(DataTable table)
-      {
+      public BlueTableSchema(DataTable table) {
          myIndexByColOrder = new ColumnInfo[table.Columns.Count];
          myTableName = table.TableName;
-         foreach (DataColumn col in table.Columns)
-         {
+         foreach (DataColumn col in table.Columns) {
             ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
             myIndexByColName[ci.colName] = ci;
             myIndexByColOrder[ci.colNo] = ci;
          }
       }
 
-      public BlueTableSchema(ColumnInfo[] cis, string tableName)
-      {
+      public BlueTableSchema(ColumnInfo[] cis, string tableName) {
          myConstruct(cis, tableName);
       }
 
-      private void myConstruct(ColumnInfo[] cis, string tableName)
-      {
+      private void myConstruct(ColumnInfo[] cis, string tableName) {
          myIndexByColOrder = new ColumnInfo[cis.Length];
          myTableName = tableName;
-         foreach (ColumnInfo ci in cis)
-         {
+         foreach (ColumnInfo ci in cis) {
             myIndexByColOrder[ci.colNo] = ci;
             myIndexByColName[ci.colName] = ci;
          }
@@ -169,15 +156,13 @@ namespace BCG
       /// </summary>
       /// <param name="tableName">the table name</param>
       /// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
-      public BlueTableSchema(string tableName, Type fromType)
-      {
+      public BlueTableSchema(string tableName, Type fromType) {
          List<ColumnInfo> cis = new List<ColumnInfo>();
          int i = 0;
          // we sort the field name so that we have a deterministic order given 
          // the same type. 
          bool foundId = false;
-         foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; }))
-         {
+         foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) {
             if (fi.Name.ToLower().Equals("id"))
                foundId = true;
             cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
@@ -188,32 +173,25 @@ namespace BCG
          myConstruct(cis.ToArray(), tableName);
       }
 
-      public ColumnInfo this[string colName]
-      {
-         get
-         {
+      public ColumnInfo this[string colName] {
+         get {
             object retVal = myIndexByColName[colName];
             return retVal != null ? retVal as ColumnInfo : null;
          }
       }
 
-      public ColumnInfo this[int colNo]
-      {
+      public ColumnInfo this[int colNo] {
          get { return myIndexByColOrder[colNo]; }
       }
 
       /// <summary>
       /// The returned Array is sorted according to the Column No
       /// </summary>
-      public string[] columnNames
-      {
-         get
-         {
-            if (myTableColNames == null)
-            {
+      public string[] columnNames {
+         get {
+            if (myTableColNames == null) {
                myTableColNames = new string[myIndexByColName.Count];
-               foreach (DictionaryEntry de in myIndexByColName)
-               {
+               foreach (DictionaryEntry de in myIndexByColName) {
                   ColumnInfo temp = (ColumnInfo)de.Value;
                   myTableColNames[temp.colNo] = temp.colName;
                }
@@ -225,8 +203,7 @@ namespace BCG
       /// <summary>
       /// The returned Array is sorted according to the Column No
       /// </summary>
-      public ColumnInfo[] columnInfo
-      {
+      public ColumnInfo[] columnInfo {
          get { return myIndexByColOrder; }
       }
 
@@ -234,49 +211,30 @@ namespace BCG
 
 
       #region ISerializedType methods
-      public void serialize(BinaryWriter bw)
-      {
+      public void serialize(BinaryWriter bw) {
          bw.Write(columnNames.Length);
          bw.Write(tableName);
-         foreach (ColumnInfo ci in columnInfo)
-         {
+         foreach (ColumnInfo ci in columnInfo) {
             bw.Write(ci.colName);
             bw.Write(ci.colType.ToString());
-            if (ci.isConstant)
-            {
-               bw.Write(true);
-               BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
-            }
-            else
             {
                bw.Write(false);
             }
          }
       }
 
-      public void deserialze(BinaryReader br)
-      {
+      public void deserialze(BinaryReader br) {
          ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
          string name = br.ReadString();
-         for (int i = 0; i < cis.Length; i++)
-         {
+         for (int i = 0; i < cis.Length; i++) {
             string colName = br.ReadString();
             Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
-            bool isConstant = br.ReadBoolean();
-            if (isConstant)
-            {
-               cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t));
-            }
-            else
-            {
-               cis[i] = new ColumnInfo(colName, i, t);
-            }
+            cis[i] = new ColumnInfo(colName, i, t);
          }
          myConstruct(cis, name);
       }
 
-      public int getSerializedSize()
-      {
+      public int getSerializedSize() {
          throw new NotImplementedException("Feature for provided constant values to" +
             "non-primitive types in schema, not provided as of now");
       }

src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+
+using BCG;
+
+namespace bts {
+   public class CircularBufferTable : IBlueRowCollection {
+      private BlueTable _table;
+      private List<IBlueRow> _ordered_list = new List<IBlueRow>();
+      private int _buffer_sz;
+      public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) {
+         _table = new BlueTable(schema);
+         _buffer_sz = buffer_sz;
+      }
+
+      // public OrderedBlueTable(List<IBlueRow> rows, int buffer_sz) {
+      //    _buffer_sz = buffer_sz;
+      //    foreach (IBlueRow r in rows)
+      //       add_row(r);
+      // }
+
+      // returns the row that might need to be deleted to keep
+      // the circular buffer intact
+      public IBlueRow add_row(IBlueRow row) {
+         _ordered_list.Add(row);
+         _table.addRow(row);
+         if (_ordered_list.Count > _buffer_sz) {
+            IBlueRow rv = _ordered_list.ElementAt(0);
+            _ordered_list.RemoveAt(0);
+            _table.deleteRow(rv);
+            return rv;
+         }
+         return null;
+      }
+
+      public IBlueRow this[int id] {
+         get {
+            return _table[id];
+         }
+      }
+      public IBlueTableSchema schema {
+         get {
+            return _table.schema;
+         }
+      }
+      public bool containsID(int id) {
+         return _table.containsID(id);
+      }
+
+      public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); }
+      public int Count { get { return _table.Count; } }
+      public bool IsSynchronized { get { return false; } }
+      public object SyncRoot { get { return this; } }
+      public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); }
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,56 +1,25 @@
 using System;
 
-namespace BCG
-{
+namespace BCG {
    /// <summary>
    /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
    /// </summary>
    [Serializable]
-   public class ColumnInfo : IComparable
-   {
-      public static bool isAllowedType (Type t)
-      {
-         return 
-            t.Equals (typeof (System.Int32)) ||
-            t.Equals (typeof (System.Int64)) ||
-            t.Equals (typeof (System.Double)) ||
-            t.Equals (typeof (System.String)) ||
-            t.Equals (typeof (System.DateTime)) ? true :
-            false;
-      }
+   public class ColumnInfo : IComparable {
 
       //the default column info does not cach values in a lookup table, therefore it is in group 0
-      public ColumnInfo(string colName, int colNo, Type colType)
-         : this(colName, colNo, colType, 0)
-      {
-
-
-      }
-
-      public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
-      {
+      public ColumnInfo(string colName, int colNo, Type colType) {
          myColName = colName;
          myColNo = colNo;
          myColType = colType;
-         myLookupTableGroup = colGroup;
-         myColValue = null;
+      }
+
+      public ColumnInfo(string col_name, int col_no, TypeCode type_code)
+         : this(col_name, col_no, BlueDataUtility.get_type(type_code)) {
       }
 
 
-      public ColumnInfo (string colName, int colNo, object colValue)
-         : this (colName, colNo, colValue.GetType())  {myColValue = colValue == null ? DBNull.Value : colValue;}
-
-      public ColumnInfo (string colName, int colNo, Type colType, object colValue)
-         : this (colName, colNo, colType) 
-      {
-
-         myColValue = colValue == null ? DBNull.Value : colValue;
-      
-      }
-
-
-      public int CompareTo (object obj)
-      {
+      public int CompareTo(object obj) {
          int otherColNo = ((ColumnInfo)(obj)).colNo;
          if (myColNo < otherColNo)
             return -1;
@@ -59,37 +28,29 @@ namespace BCG
          return 0;
       }
 
-      //public bool isCalculated{get{return myExpression != null;}}
+      public override bool Equals(object obj) {
+         if (obj is ColumnInfo) {
+            ColumnInfo other = obj as ColumnInfo;
+            return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
+         }
+         return false;
+      }
 
-      public bool isConstant {get{return myColValue != null;}}
+      public override string ToString() {
+         return colName + " " + colNo + " " + colType;
+      }
+
+
+      // for backwards compatibility to allow build
+      public bool isConstant { get { return false; } }
 
       private int myColNo;
       private string myColName;
       private Type myColType;
-      //private string myExpression;
-      private object myColValue;
 
-      //no table will ever have more than 65,536 columns
-      //if it does, we are in serious trouble
-      private short myLookupTableGroup;
+      public int colNo { get { return myColNo; } }
+      public string colName { get { return myColName; } }
+      public Type colType { get { return myColType; } }
 
-      public int colNo{get{return myColNo;}}
-      public string colName {get{return myColName;}}
-      public Type colType {get{return myColType;}}
-
-
-      //this value corresponds to how the data in the column is serialized. 
-      //if the group number is 0, the values are not stored in a table
-      //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
-      //are stored together in a lookup table
-
-      
-      public short lookupTableGroup
-      {
-         get { return myLookupTableGroup; }
-         set { myLookupTableGroup = value; }
-      }
-      //public string expression {get{return myExpression;}}
-      public object colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs

@@ -1,252 +1,252 @@
-using System;
-using System.Collections;
-//using BCG.Utilities;
+// using System;
+// using System.Collections;
+// //using BCG.Utilities;
 
-namespace BCG
-{
-   /// <summary>
-   /// Provides a multiple indexing mechanism, required to increase the peformance of the streams
-   /// </summary>
-   public class BlueTableIndexException : Exception
-   {
-      public BlueTableIndexException(string msg) : base(msg) { }
-   }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// Provides a multiple indexing mechanism, required to increase the peformance of the streams
+//    /// </summary>
+//    public class BlueTableIndexException : Exception
+//    {
+//       public BlueTableIndexException(string msg) : base(msg) { }
+//    }
 
-   public interface IBlueTableIndex
-   {
-      void addRow(IBlueRow row);
-      void deleteRowOnPrimaryKey(object pkValue);
-      void deleteRowOnPrimaryKey(IBlueRow row);
-      void deleteRowOnBothKeys(object pkValue, object skValue);
-      void deleteRowOnBothKeys(IBlueRow row);
-      ArrayList searchRowOnPrimaryKey(object pkValue);
-      ArrayList searchRowOnPrimaryKey(IBlueRow row);
-      ArrayList searchRowOnBothKeys(object pkValue, object skValue);
-      ArrayList searchRowOnBothKeys(IBlueRow row);
+//    public interface IBlueTableIndex
+//    {
+//       void addRow(IBlueRow row);
+//       void deleteRowOnPrimaryKey(object pkValue);
+//       void deleteRowOnPrimaryKey(IBlueRow row);
+//       void deleteRowOnBothKeys(object pkValue, object skValue);
+//       void deleteRowOnBothKeys(IBlueRow row);
+//       ArrayList searchRowOnPrimaryKey(object pkValue);
+//       ArrayList searchRowOnPrimaryKey(IBlueRow row);
+//       ArrayList searchRowOnBothKeys(object pkValue, object skValue);
+//       ArrayList searchRowOnBothKeys(IBlueRow row);
 
 
-   }
+//    }
 
-   /*
-   public class BlueTableDoubleIndex : IBlueTableIndex
-   {
-      private IBlueRowCollection myTable;         
-      private string myPrimaryIndex;
-      private string mySecondaryIndex;
-      private Hashtable myHt = new Hashtable ();
-      
-      /// <summary>
-      /// Builds an index with 2 columns of the table acting as primary index and secondary index
-      /// respectively
-      /// </summary>
-      /// <param name="table">
-      /// table to index
-      /// </param>
-      /// <param name="primaryIndex">
-      /// primary key to index upon
-      /// </param>
-      /// <param name="secondaryIndex">
-      /// column serving as the secondary key
-      /// </param>
-      public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
-      {
-         myTable = table;
-         myPrimaryIndex = primaryIndex;
-         mySecondaryIndex = secondaryIndex;
-         myInit ();
-      }
-      
-      private void myInit()
-      {
-         foreach (IBlueRow row in myTable)
-            addRow (row);
-      }
+//    /*
+//    public class BlueTableDoubleIndex : IBlueTableIndex
+//    {
+//       private IBlueRowCollection myTable;         
+//       private string myPrimaryIndex;
+//       private string mySecondaryIndex;
+//       private Hashtable myHt = new Hashtable ();
 
-      /// <summary>
-      /// To add a row to the index
-      /// </summary>
-      /// <param name="row">
-      /// row to be added
-      /// </param>
-      public void addRow (IBlueRow row)
-      {
-         if (myHt[row[myPrimaryIndex]] != null)
-            ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
-         else
-         {
-            TimeManager.start ();
-            Hashtable ht = new Hashtable();
-            TimeManager.stop ();
-            ht[row[mySecondaryIndex]] = row;
-            myHt[row[myPrimaryIndex]] = ht;
-         }
-      }
+//       /// <summary>
+//       /// Builds an index with 2 columns of the table acting as primary index and secondary index
+//       /// respectively
+//       /// </summary>
+//       /// <param name="table">
+//       /// table to index
+//       /// </param>
+//       /// <param name="primaryIndex">
+//       /// primary key to index upon
+//       /// </param>
+//       /// <param name="secondaryIndex">
+//       /// column serving as the secondary key
+//       /// </param>
+//       public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
+//       {
+//          myTable = table;
+//          myPrimaryIndex = primaryIndex;
+//          mySecondaryIndex = secondaryIndex;
+//          myInit ();
+//       }
 
-      /// <summary>
-      /// If index or the primary key has already been deleted appropriate exceptions are thrown
-      /// </summary>
-      /// <param name="row"></param>
-      public void deleteRow (IBlueRow row)
-      {
-         if (myHt[row[myPrimaryIndex]] == null)
-         {
-            throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
-         }
-         ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
-         if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
-            myHt.Remove(row[myPrimaryIndex]);
-      }
+//       private void myInit()
+//       {
+//          foreach (IBlueRow row in myTable)
+//             addRow (row);
+//       }
 
-      /// <summary>
-      /// Returns an array of DataRow that match the primary key being passed as an argument
-      /// </summary>
-      /// <param name="o">
-      /// Primary Key of the table
-      /// </param>
-      /// <returns>
-      /// Array of rows matching the criteria
-      /// </returns>
-      public IBlueRow[] findValueFromPrimaryKey (object o)
-      {
-         object oht = myHt[o];
-         if (oht == null)
-            return new IBlueRow[0];
-         Hashtable ht = (Hashtable)oht;
-         IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
-         ht.Values.CopyTo(rows, 0);
-         return rows;
-      }
+//       /// <summary>
+//       /// To add a row to the index
+//       /// </summary>
+//       /// <param name="row">
+//       /// row to be added
+//       /// </param>
+//       public void addRow (IBlueRow row)
+//       {
+//          if (myHt[row[myPrimaryIndex]] != null)
+//             ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
+//          else
+//          {
+//             TimeManager.start ();
+//             Hashtable ht = new Hashtable();
+//             TimeManager.stop ();
+//             ht[row[mySecondaryIndex]] = row;
+//             myHt[row[myPrimaryIndex]] = ht;
+//          }
+//       }
+
+//       /// <summary>
+//       /// If index or the primary key has already been deleted appropriate exceptions are thrown
+//       /// </summary>
+//       /// <param name="row"></param>
+//       public void deleteRow (IBlueRow row)
+//       {
+//          if (myHt[row[myPrimaryIndex]] == null)
+//          {
+//             throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
+//          }
+//          ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
+//          if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
+//             myHt.Remove(row[myPrimaryIndex]);
+//       }
+
+//       /// <summary>
+//       /// Returns an array of DataRow that match the primary key being passed as an argument
+//       /// </summary>
+//       /// <param name="o">
+//       /// Primary Key of the table
+//       /// </param>
+//       /// <returns>
+//       /// Array of rows matching the criteria
+//       /// </returns>
+//       public IBlueRow[] findValueFromPrimaryKey (object o)
+//       {
+//          object oht = myHt[o];
+//          if (oht == null)
+//             return new IBlueRow[0];
+//          Hashtable ht = (Hashtable)oht;
+//          IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
+//          ht.Values.CopyTo(rows, 0);
+//          return rows;
+//       }
 
 
-      /// <summary>
-      /// returns as array of the primary keys
-      /// </summary>
-      /// <returns>
-      /// object[] of primary keys
-      /// </returns>
-      public Array getPrimaryKeys()
-      {
-         object[] retval = new object [myHt.Keys.Count];
-         myHt.Keys.CopyTo (retval, 0);
-         return retval;
-      }
-   }
-   */
+//       /// <summary>
+//       /// returns as array of the primary keys
+//       /// </summary>
+//       /// <returns>
+//       /// object[] of primary keys
+//       /// </returns>
+//       public Array getPrimaryKeys()
+//       {
+//          object[] retval = new object [myHt.Keys.Count];
+//          myHt.Keys.CopyTo (retval, 0);
+//          return retval;
+//       }
+//    }
+//    */
 
-   /// <summary>
-   /// Provides a 1-Level indexing structure.
-   /// </summary>
-   public class BlueTableSingleIndex : IBlueTableIndex
-   {
-      string myPrimaryKey;
-      string mySecondaryKey;
-      Hashtable myHt = new Hashtable();
+//    /// <summary>
+//    /// Provides a 1-Level indexing structure.
+//    /// </summary>
+//    public class BlueTableSingleIndex : IBlueTableIndex
+//    {
+//       string myPrimaryKey;
+//       string mySecondaryKey;
+//       Hashtable myHt = new Hashtable();
 
-      public BlueTableSingleIndex(string primaryKey, string secondaryKey)
-      {
-         myPrimaryKey = primaryKey;
-         mySecondaryKey = secondaryKey;
-      }
+//       public BlueTableSingleIndex(string primaryKey, string secondaryKey)
+//       {
+//          myPrimaryKey = primaryKey;
+//          mySecondaryKey = secondaryKey;
+//       }
 
-      /// <summary>
-      /// The primary key is the indexing key, and the secondary key is used as the key for
-      /// searching and hence deletion
-      /// </summary>
-      /// <param name="table"></param>
-      /// <param name="primaryKey"></param>
-      /// <param name="secondaryKey"></param>
-      public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
-         : this(primaryKey, secondaryKey)
-      {
-         foreach (IBlueRow row in table)
-            addRow(row);
-      }
+//       /// <summary>
+//       /// The primary key is the indexing key, and the secondary key is used as the key for
+//       /// searching and hence deletion
+//       /// </summary>
+//       /// <param name="table"></param>
+//       /// <param name="primaryKey"></param>
+//       /// <param name="secondaryKey"></param>
+//       public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
+//          : this(primaryKey, secondaryKey)
+//       {
+//          foreach (IBlueRow row in table)
+//             addRow(row);
+//       }
 
 
 
-      /// <summary>
-      /// Adds row to the indexing new structure
-      /// </summary>
-      /// <param name="row"></param>
-      public void addRow(IBlueRow row)
-      {
-         if (myHt[row[myPrimaryKey]] == null)
-            myHt[row[myPrimaryKey]] = new ArrayList();
-         ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
-      }
+//       /// <summary>
+//       /// Adds row to the indexing new structure
+//       /// </summary>
+//       /// <param name="row"></param>
+//       public void addRow(IBlueRow row)
+//       {
+//          if (myHt[row[myPrimaryKey]] == null)
+//             myHt[row[myPrimaryKey]] = new ArrayList();
+//          ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
+//       }
 
-      /// <summary>
-      /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
-      /// </summary>
-      /// <param name="row"></param>
-      /// <returns></returns>
-      public ArrayList searchRowOnPrimaryKey(IBlueRow row)
-      {
-         return searchRowOnPrimaryKey(row[myPrimaryKey]);
-      }
+//       /// <summary>
+//       /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
+//       /// </summary>
+//       /// <param name="row"></param>
+//       /// <returns></returns>
+//       public ArrayList searchRowOnPrimaryKey(IBlueRow row)
+//       {
+//          return searchRowOnPrimaryKey(row[myPrimaryKey]);
+//       }
 
-      public ArrayList searchRowOnPrimaryKey(object pkValue)
-      {
-         if (myHt[pkValue] == null)
-            return new ArrayList();
-         return (ArrayList)myHt[pkValue];
-      }
+//       public ArrayList searchRowOnPrimaryKey(object pkValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             return new ArrayList();
+//          return (ArrayList)myHt[pkValue];
+//       }
 
-      /// <summary>
-      /// Gets rows based on the primary AND the secondary key
-      /// </summary>
-      /// <param name="row"></param>
-      /// <returns></returns>
-      public ArrayList searchRowOnBothKeys(IBlueRow row)
-      {
-         return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
-      }
+//       /// <summary>
+//       /// Gets rows based on the primary AND the secondary key
+//       /// </summary>
+//       /// <param name="row"></param>
+//       /// <returns></returns>
+//       public ArrayList searchRowOnBothKeys(IBlueRow row)
+//       {
+//          return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
+//       }
 
-      public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
-      {
-         if (myHt[pkValue] == null)
-            return new ArrayList();
-         ArrayList matchingList = (ArrayList)myHt[pkValue];
-         ArrayList retVal = new ArrayList();
-         foreach (IBlueRow r in matchingList)
-            if (r[mySecondaryKey].Equals(skValue))
-               retVal.Add(r);
-         return retVal;
-      }
+//       public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             return new ArrayList();
+//          ArrayList matchingList = (ArrayList)myHt[pkValue];
+//          ArrayList retVal = new ArrayList();
+//          foreach (IBlueRow r in matchingList)
+//             if (r[mySecondaryKey].Equals(skValue))
+//                retVal.Add(r);
+//          return retVal;
+//       }
 
 
-      /// <summary>
-      /// Deletes rows based on the primary key only
-      /// </summary>
-      /// <param name="row"></param>
-      public void deleteRowOnPrimaryKey(IBlueRow row)
-      {
-         deleteRowOnPrimaryKey(row[myPrimaryKey]);
-      }
+//       /// <summary>
+//       /// Deletes rows based on the primary key only
+//       /// </summary>
+//       /// <param name="row"></param>
+//       public void deleteRowOnPrimaryKey(IBlueRow row)
+//       {
+//          deleteRowOnPrimaryKey(row[myPrimaryKey]);
+//       }
 
-      public void deleteRowOnPrimaryKey(object pkValue)
-      {
-         if (myHt[pkValue] == null)
-            throw new BlueTableIndexException("Key to be deleted was not found in the index");
-         myHt.Remove(pkValue);
-      }
+//       public void deleteRowOnPrimaryKey(object pkValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             throw new BlueTableIndexException("Key to be deleted was not found in the index");
+//          myHt.Remove(pkValue);
+//       }
 
-      public void deleteRowOnBothKeys(IBlueRow row)
-      {
-         deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
-      }
+//       public void deleteRowOnBothKeys(IBlueRow row)
+//       {
+//          deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
+//       }
 
-      public void deleteRowOnBothKeys(object pkValue, object skValue)
-      {
-         if (myHt[pkValue] == null)
-            throw new BlueTableIndexException("Key to be deleted was not found in the index");
-         ArrayList matchingList = (ArrayList)myHt[pkValue];
-         ArrayList aux = new ArrayList();
-         foreach (IBlueRow r in matchingList)
-            if (r[mySecondaryKey].Equals(skValue))
-               aux.Add(r);
-         foreach (IBlueRow r in aux)
-            matchingList.Remove(r);
-      }
-   }
-}
+//       public void deleteRowOnBothKeys(object pkValue, object skValue)
+//       {
+//          if (myHt[pkValue] == null)
+//             throw new BlueTableIndexException("Key to be deleted was not found in the index");
+//          ArrayList matchingList = (ArrayList)myHt[pkValue];
+//          ArrayList aux = new ArrayList();
+//          foreach (IBlueRow r in matchingList)
+//             if (r[mySecondaryKey].Equals(skValue))
+//                aux.Add(r);
+//          foreach (IBlueRow r in aux)
+//             matchingList.Remove(r);
+//       }
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs

@@ -1,324 +1,324 @@
-using System;
-using System.Data;
-using System.Collections;
-using System.IO;
-using System.Reflection;
-using System.Collections.Generic;
-using System.Linq;
+// using System;
+// using System.Data;
+// using System.Collections;
+// using System.IO;
+// using System.Reflection;
+// using System.Collections.Generic;
+// using System.Linq;
 
-using BCG.Utilities.General;
+// using BCG.Utilities.General;
 
-namespace BCG
-{
-   /// <summary>
-   /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
-   /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
-   /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
-   /// the respective data, sorted according to the ColumnOrder.
-   /// </summary>
-   [Serializable]
-   public class LookupTableSchema : IBlueTableSchema
-   {
-      public class LookupTableSchemaSerializationHelper
-      {
-         public static object parse(Type t, string data)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return Boolean.Parse(data);
-            else if (t == BlueDataValidTypes.BYTE)
-               return Byte.Parse(data);
-            else if (t == BlueDataValidTypes.CHAR)
-               return char.Parse(data);
-            else if (t == BlueDataValidTypes.DATETIME)
-               return DateTime.Parse(data);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return Double.Parse(data);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return Decimal.Parse(data);
-            else if (t == BlueDataValidTypes.INT)
-               return int.Parse(data);
-            else if (t == BlueDataValidTypes.LONG)
-               return long.Parse(data);
-            else if (t == BlueDataValidTypes.SBYTE)
-               return SByte.Parse(data);
-            else if (t == BlueDataValidTypes.STRING)
-               return data;
-            else if (t == BlueDataValidTypes.UINT)
-               return uint.Parse(data);
-            else if (t == BlueDataValidTypes.ULONG)
-               return ulong.Parse(data);
-            else if (t == BlueDataValidTypes.SINGLE)
-               return Single.Parse(data);
-            else
-               throw new ApplicationException("Can not be parsed");
-         }
-         public static void serialize(BinaryWriter bw, Type t, object o)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               bw.Write((bool)o);
-            else if (t == BlueDataValidTypes.BYTE)
-               bw.Write((byte)o);
-            else if (t == BlueDataValidTypes.CHAR)
-               bw.Write((char)o);
-            else if (t == BlueDataValidTypes.DATETIME)
-               bw.Write(((DateTime)o).Ticks);
-            else if (t == BlueDataValidTypes.DOUBLE)
-               bw.Write((double)o);
-            else if (t == BlueDataValidTypes.DECIMAL)
-               bw.Write((decimal)o);
-            else if (t == BlueDataValidTypes.INT)
-               bw.Write((int)o);
-            else if (t == BlueDataValidTypes.LONG)
-               bw.Write((long)o);
-            else if (t == BlueDataValidTypes.SBYTE)
-               bw.Write((sbyte)o);
-            else if (t == BlueDataValidTypes.STRING)
-               bw.Write(o as string);
-            else if (t == BlueDataValidTypes.SINGLE)
-               bw.Write((Single)o);
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-            {
-               byte[] bs = o as byte[];
-               bw.Write(bs.Length);
-               bw.Write(bs);
-            }
-            else if (t == BlueDataValidTypes.UINT)
-               bw.Write((uint)o);
-            else if (t == BlueDataValidTypes.ULONG)
-               bw.Write((ulong)o);
-            else
-               throw new ApplicationException("Unsupported type");
-         }
+// namespace BCG
+// {
+//    /// <summary>
+//    /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
+//    /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
+//    /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
+//    /// the respective data, sorted according to the ColumnOrder.
+//    /// </summary>
+//    [Serializable]
+//    public class LookupTableSchema : IBlueTableSchema
+//    {
+//       public class LookupTableSchemaSerializationHelper
+//       {
+//          public static object parse(Type t, string data)
+//          {
+//             if (t == BlueDataValidTypes.BOOL)
+//                return Boolean.Parse(data);
+//             else if (t == BlueDataValidTypes.BYTE)
+//                return Byte.Parse(data);
+//             else if (t == BlueDataValidTypes.CHAR)
+//                return char.Parse(data);
+//             else if (t == BlueDataValidTypes.DATETIME)
+//                return DateTime.Parse(data);
+//             else if (t == BlueDataValidTypes.DOUBLE)
+//                return Double.Parse(data);
+//             else if (t == BlueDataValidTypes.DECIMAL)
+//                return Decimal.Parse(data);
+//             else if (t == BlueDataValidTypes.INT)
+//                return int.Parse(data);
+//             else if (t == BlueDataValidTypes.LONG)
+//                return long.Parse(data);
+//             else if (t == BlueDataValidTypes.SBYTE)
+//                return SByte.Parse(data);
+//             else if (t == BlueDataValidTypes.STRING)
+//                return data;
+//             else if (t == BlueDataValidTypes.UINT)
+//                return uint.Parse(data);
+//             else if (t == BlueDataValidTypes.ULONG)
+//                return ulong.Parse(data);
+//             else if (t == BlueDataValidTypes.SINGLE)
+//                return Single.Parse(data);
+//             else
+//                throw new ApplicationException("Can not be parsed");
+//          }
+//          public static void serialize(BinaryWriter bw, Type t, object o)
+//          {
+//             if (t == BlueDataValidTypes.BOOL)
+//                bw.Write((bool)o);
+//             else if (t == BlueDataValidTypes.BYTE)
+//                bw.Write((byte)o);
+//             else if (t == BlueDataValidTypes.CHAR)
+//                bw.Write((char)o);
+//             else if (t == BlueDataValidTypes.DATETIME)
+//                bw.Write(((DateTime)o).Ticks);
+//             else if (t == BlueDataValidTypes.DOUBLE)
+//                bw.Write((double)o);
+//             else if (t == BlueDataValidTypes.DECIMAL)
+//                bw.Write((decimal)o);
+//             else if (t == BlueDataValidTypes.INT)
+//                bw.Write((int)o);
+//             else if (t == BlueDataValidTypes.LONG)
+//                bw.Write((long)o);
+//             else if (t == BlueDataValidTypes.SBYTE)
+//                bw.Write((sbyte)o);
+//             else if (t == BlueDataValidTypes.STRING)
+//                bw.Write(o as string);
+//             else if (t == BlueDataValidTypes.SINGLE)
+//                bw.Write((Single)o);
+//             else if (t == BlueDataValidTypes.BYTEARRAY)
+//             {
+//                byte[] bs = o as byte[];
+//                bw.Write(bs.Length);
+//                bw.Write(bs);
+//             }
+//             else if (t == BlueDataValidTypes.UINT)
+//                bw.Write((uint)o);
+//             else if (t == BlueDataValidTypes.ULONG)
+//                bw.Write((ulong)o);
+//             else
+//                throw new ApplicationException("Unsupported type");
+//          }
 
-         public static object deserialize(BinaryReader br, Type t)
-         {
-            if (t == BlueDataValidTypes.BOOL)
-               return br.ReadBoolean();
-            else if (t == BlueDataValidTypes.BYTE)
-               return br.ReadByte();
-            else if (t == BlueDataValidTypes.CHAR)
-               return br.ReadChar();
-            else if (t == BlueDataValidTypes.DATETIME)
-               return new DateTime(br.ReadInt64());
-            else if (t == BlueDataValidTypes.DOUBLE)
-               return br.ReadDouble();
-            else if (t == BlueDataValidTypes.DECIMAL)
-               return br.ReadDecimal();
-            else if (t == BlueDataValidTypes.INT)
-               return br.ReadInt32();
-            else if (t == BlueDataValidTypes.LONG)
-               return br.ReadInt64();
-            else if (t == BlueDataValidTypes.SBYTE)
-               return br.ReadSByte();
-            else if (t == BlueDataValidTypes.STRING)
-               return br.ReadString();
-            else if (t == BlueDataValidTypes.BYTEARRAY)
-               return br.ReadBytes(br.ReadInt32());
-            else if (t == BlueDataValidTypes.UINT)
-               return br.ReadUInt32();
-            else if (t == BlueDataValidTypes.ULONG)
-               return br.ReadUInt64();
-            else if (t == BlueDataValidTypes.SINGLE)
-               return br.ReadSingle();
-            else
-               throw new ApplicationException("Unsupported type");
-         }
-      }
+//          public static object deserialize(BinaryReader br, Type t)
+//          {
+//             if (t == BlueDataValidTypes.BOOL)
+//                return br.ReadBoolean();
+//             else if (t == BlueDataValidTypes.BYTE)
+//                return br.ReadByte();
+//             else if (t == BlueDataValidTypes.CHAR)
+//                return br.ReadChar();
+//             else if (t == BlueDataValidTypes.DATETIME)
+//                return new DateTime(br.ReadInt64());
+//             else if (t == BlueDataValidTypes.DOUBLE)
+//                return br.ReadDouble();
+//             else if (t == BlueDataValidTypes.DECIMAL)
+//                return br.ReadDecimal();
+//             else if (t == BlueDataValidTypes.INT)
+//                return br.ReadInt32();
+//             else if (t == BlueDataValidTypes.LONG)
+//                return br.ReadInt64();
+//             else if (t == BlueDataValidTypes.SBYTE)
+//                return br.ReadSByte();
+//             else if (t == BlueDataValidTypes.STRING)
+//                return br.ReadString();
+//             else if (t == BlueDataValidTypes.BYTEARRAY)
+//                return br.ReadBytes(br.ReadInt32());
+//             else if (t == BlueDataValidTypes.UINT)
+//                return br.ReadUInt32();
+//             else if (t == BlueDataValidTypes.ULONG)
+//                return br.ReadUInt64();
+//             else if (t == BlueDataValidTypes.SINGLE)
+//                return br.ReadSingle();
+//             else
+//                throw new ApplicationException("Unsupported type");
+//          }
+//       }
 
-      protected Hashtable myIndexByColName = new Hashtable();
-      protected ColumnInfo[] myIndexByColOrder;
-      private string[] myTableColNames;
-      private BlueRowLookupGroup[] myLookupGroups;
-      protected string myTableName;
+//       protected Hashtable myIndexByColName = new Hashtable();
+//       protected ColumnInfo[] myIndexByColOrder;
+//       private string[] myTableColNames;
+//       private BlueRowLookupGroup[] myLookupGroups;
+//       protected string myTableName;
 
-      public LookupTableSchema() { }
+//       public LookupTableSchema() { }
 
 
-      public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
-      {
-         myConstruct(tableName, cis, groups);
-      }
+//       public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
+//       {
+//          myConstruct(tableName, cis, groups);
+//       }
 
-      private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
-      {
-         myIndexByColOrder = new ColumnInfo[cis.Length];
-         myTableName = tableName;
-         foreach (ColumnInfo ci in cis)
-         {
-            myIndexByColOrder[ci.colNo] = ci;
-            myIndexByColName[ci.colName] = ci;
-         }
-         myLookupGroups = groups;
+//       private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
+//       {
+//          myIndexByColOrder = new ColumnInfo[cis.Length];
+//          myTableName = tableName;
+//          foreach (ColumnInfo ci in cis)
+//          {
+//             myIndexByColOrder[ci.colNo] = ci;
+//             myIndexByColName[ci.colName] = ci;
+//          }
+//          myLookupGroups = groups;
 
-         foreach (BlueRowLookupGroup group in groups)
-         {
-            group.buildEntrySchema(this);
+//          foreach (BlueRowLookupGroup group in groups)
+//          {
+//             group.buildEntrySchema(this);
 
 
-         }
-      }
+//          }
+//       }
 
 
 
 
-      public ColumnInfo this[string colName]
-      {
-         get
-         {
-            object retVal = myIndexByColName[colName];
-            return retVal != null ? retVal as ColumnInfo : null;
-         }
-      }
+//       public ColumnInfo this[string colName]
+//       {
+//          get
+//          {
+//             object retVal = myIndexByColName[colName];
+//             return retVal != null ? retVal as ColumnInfo : null;
+//          }
+//       }
 
-      public BlueRowLookupGroup[] lookupGroups
-      {
-         get { return myLookupGroups; }
+//       public BlueRowLookupGroup[] lookupGroups
+//       {
+//          get { return myLookupGroups; }
 
-      }
-      public ColumnInfo this[int colNo]
-      {
-         get { return myIndexByColOrder[colNo]; }
-      }
+//       }
+//       public ColumnInfo this[int colNo]
+//       {
+//          get { return myIndexByColOrder[colNo]; }
+//       }
 
-      /// <summary>
-      /// The returned Array is sorted according to the Column No
-      /// </summary>
-      public string[] columnNames
-      {
-         get
-         {
-            if (myTableColNames == null)
-            {
-               myTableColNames = new string[myIndexByColName.Count];
-               foreach (DictionaryEntry de in myIndexByColName)
-               {
-                  ColumnInfo temp = (ColumnInfo)de.Value;
-                  myTableColNames[temp.colNo] = temp.colName;
-               }
-            }
-            return myTableColNames;
-         }
-      }
+//       /// <summary>
+//       /// The returned Array is sorted according to the Column No
+//       /// </summary>
+//       public string[] columnNames
+//       {
+//          get
+//          {
+//             if (myTableColNames == null)
+//             {
+//                myTableColNames = new string[myIndexByColName.Count];
+//                foreach (DictionaryEntry de in myIndexByColName)
+//                {
+//                   ColumnInfo temp = (ColumnInfo)de.Value;
+//                   myTableColNames[temp.colNo] = temp.colName;
+//                }
+//             }
+//             return myTableColNames;
+//          }
+//       }
 
-      /// <summary>
-      /// The returned Array is sorted according to the Column No
-      /// </summary>
-      public ColumnInfo[] columnInfo
-      {
-         get { return myIndexByColOrder; }
-      }
+//       /// <summary>
+//       /// The returned Array is sorted according to the Column No
+//       /// </summary>
+//       public ColumnInfo[] columnInfo
+//       {
+//          get { return myIndexByColOrder; }
+//       }
 
-      public string tableName { get { return myTableName; } }
+//       public string tableName { get { return myTableName; } }
 
 
-      #region ISerializedType methods
-      public void serialize(BinaryWriter bw)
-      {
-         bw.Write(columnNames.Length);
-         bw.Write(tableName);
+//       #region ISerializedType methods
+//       public void serialize(BinaryWriter bw)
+//       {
+//          bw.Write(columnNames.Length);
+//          bw.Write(tableName);
 
-         //first, write the column names
-         foreach (ColumnInfo ci in columnInfo)
-         {
-            bw.Write(ci.colName);
-            bw.Write(ci.colType.ToString());
-            if (ci.isConstant)
-            {
-               bw.Write(true);
-               LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
-            }
-            else
-            {
-               bw.Write(false);
-               bw.Write(ci.lookupTableGroup);
-            }
+//          //first, write the column names
+//          foreach (ColumnInfo ci in columnInfo)
+//          {
+//             bw.Write(ci.colName);
+//             bw.Write(ci.colType.ToString());
+//             if (ci.isConstant)
+//             {
+//                bw.Write(true);
+//                LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
+//             }
+//             else
+//             {
+//                bw.Write(false);
+//                bw.Write(ci.lookupTableGroup);
+//             }
 
-         }
+//          }
 
-         //this is here for error checking
-         bw.Write(myLookupGroups.Length);
-         //now, write the size of each lookup table group
-         foreach (BlueRowLookupGroup group in myLookupGroups)
-         {
-            bw.Write(group.groupNumber);
-            bw.Write(group.tableSize);
-            bw.Write(group.groupColNames.Length);
-            foreach (string colName in group.groupColNames)
-            {
-               //write the index of this column in the table
-               bw.Write(this[colName].colNo);
+//          //this is here for error checking
+//          bw.Write(myLookupGroups.Length);
+//          //now, write the size of each lookup table group
+//          foreach (BlueRowLookupGroup group in myLookupGroups)
+//          {
+//             bw.Write(group.groupNumber);
+//             bw.Write(group.tableSize);
+//             bw.Write(group.groupColNames.Length);
+//             foreach (string colName in group.groupColNames)
+//             {
+//                //write the index of this column in the table
+//                bw.Write(this[colName].colNo);
 
-            }
+//             }
 
 
-         }
+//          }
 
 
 
-      }
+//       }
 
-      public void deserialze(BinaryReader br)
-      {
-         int numCols = br.ReadInt32();
-         //these are the final output columns that will be created
-         ColumnInfo[] columnInfos = new ColumnInfo[numCols];
+//       public void deserialze(BinaryReader br)
+//       {
+//          int numCols = br.ReadInt32();
+//          //these are the final output columns that will be created
+//          ColumnInfo[] columnInfos = new ColumnInfo[numCols];
 
-         Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
+//          Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
 
 
-         string tableName = br.ReadString();
-         for (int colNo = 0; colNo < numCols; ++colNo)
-         {
-            string colName = br.ReadString();
-            Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
-            bool colIsConst = br.ReadBoolean();
-            if (colIsConst)
-            {
-               columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
+//          string tableName = br.ReadString();
+//          for (int colNo = 0; colNo < numCols; ++colNo)
+//          {
+//             string colName = br.ReadString();
+//             Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
+//             bool colIsConst = br.ReadBoolean();
+//             if (colIsConst)
+//             {
+//                columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
 
-            }
-            else
-            {
-               short thisColGroup = br.ReadInt16();
-               columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
-            }
-         }
+//             }
+//             else
+//             {
+//                short thisColGroup = br.ReadInt16();
+//                columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
+//             }
+//          }
 
-         int numGroups = br.ReadInt32();
-         BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
+//          int numGroups = br.ReadInt32();
+//          BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
 
-         for (int group = 0; group < numGroups; ++group)
-         {
-            short groupNo = br.ReadInt16();
-            short groupSizeInBits = br.ReadInt16();
-            int numColsInGroup = br.ReadInt32();
-            int[] groupCols = new int[numColsInGroup];
-            string[] groupColNames = new string[numColsInGroup];
-            for (int col = 0; col < numColsInGroup; ++col)
-            {
-               groupCols[col] = br.ReadInt32();
-               groupColNames[col] = columnInfos[groupCols[col]].colName;
-            }
+//          for (int group = 0; group < numGroups; ++group)
+//          {
+//             short groupNo = br.ReadInt16();
+//             short groupSizeInBits = br.ReadInt16();
+//             int numColsInGroup = br.ReadInt32();
+//             int[] groupCols = new int[numColsInGroup];
+//             string[] groupColNames = new string[numColsInGroup];
+//             for (int col = 0; col < numColsInGroup; ++col)
+//             {
+//                groupCols[col] = br.ReadInt32();
+//                groupColNames[col] = columnInfos[groupCols[col]].colName;
+//             }
 
-            //create the group
-            theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
+//             //create the group
+//             theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
 
 
-         }
+//          }
 
 
-         myConstruct(tableName, columnInfos, theGroups);
+//          myConstruct(tableName, columnInfos, theGroups);
 
 
 
-      }
+//       }
 
-      public int getSerializedSize()
-      {
-         throw new NotImplementedException("Feature for provided constant values to" +
-            "non-primitive types in schema, not provided as of now");
-      }
+//       public int getSerializedSize()
+//       {
+//          throw new NotImplementedException("Feature for provided constant values to" +
+//             "non-primitive types in schema, not provided as of now");
+//       }
 
-      #endregion
-   }
-}
+//       #endregion
+//    }
+// }

src/DataStreamSuite/DataStreamCore/BlueData/MetaDataField.cs

@@ -3,8 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BCG
-{
+namespace BCG {
    /*
       These fields are first going to be used by  BlueRay to help it determine what differnet tags mean, but eventually
     * the goal is that they will be used for all services in this system. 
@@ -12,11 +11,12 @@ namespace BCG
     * not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
     * probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
     */
-   public enum MetaDataField {   Protocol,  //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
-                                 Type,   //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
-                                 Symbol, 
-                                 Exchange,
-                                 Platform,
-                                 MarketDataType, // BAM (bid, ask, mid), last, etc
-                                 RealName }; //does this thing have a better name we could call it
+   public enum MetaDataField {
+      Protocol,       // is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
+      Type,           // if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
+      Symbol,
+      Exchange,
+      MarketDataType, // BAM (bid, ask, mid), last, etc
+      RealName
+   }; //does this thing have a better name we could call it
 }

src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs

@@ -5,8 +5,7 @@ using System.IO;
 using BCG.Utilities.General;
 //using BCG.Utilities;
 
-namespace BCG
-{
+namespace BCG {
     /// <summary>
     /// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
     /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
@@ -17,12 +16,9 @@ namespace BCG
     /// memory streams, and uses this schema to unpack the values when they are needed. 
     /// </summary>
     [Serializable]
-    public class SerializedBlueTableSchema : IBlueTableSchema
-    {
-        public class SerializedBlueTableSchemaSerializationHelper
-        {
-            public static object parse(Type t, string data)
-            {
+    public class SerializedBlueTableSchema : IBlueTableSchema {
+        public class SerializedBlueTableSchemaSerializationHelper {
+            public static object parse(Type t, string data) {
                 if (t == BlueDataValidTypes.BOOL)
                     return Boolean.Parse(data);
                 else if (t == BlueDataValidTypes.BYTE)
@@ -52,8 +48,7 @@ namespace BCG
                 else
                     throw new ApplicationException("Can not be parsed");
             }
-            public static void serialize(BinaryWriter bw, Type t, object o)
-            {
+            public static void serialize(BinaryWriter bw, Type t, object o) {
                 if (t == BlueDataValidTypes.BOOL)
                     bw.Write((bool)o);
                 else if (t == BlueDataValidTypes.BYTE)
@@ -76,13 +71,11 @@ namespace BCG
                     bw.Write(o as string);
                 else if (t == BlueDataValidTypes.SINGLE)
                     bw.Write((Single)o);
-                else if (t == BlueDataValidTypes.BYTEARRAY)
-                {
+                else if (t == BlueDataValidTypes.BYTEARRAY) {
                     byte[] bs = o as byte[];
                     bw.Write(bs.Length);
                     bw.Write(bs);
-                }
-                else if (t == BlueDataValidTypes.UINT)
+                } else if (t == BlueDataValidTypes.UINT)
                     bw.Write((uint)o);
                 else if (t == BlueDataValidTypes.ULONG)
                     bw.Write((ulong)o);
@@ -90,8 +83,7 @@ namespace BCG
                     throw new ApplicationException("Unsupported type");
             }
 
-            public static object deserialize(BinaryReader br, Type t)
-            {
+            public static object deserialize(BinaryReader br, Type t) {
                 if (t == BlueDataValidTypes.BOOL)
                     return br.ReadBoolean();
                 else if (t == BlueDataValidTypes.BYTE)
@@ -132,45 +124,37 @@ namespace BCG
         public SerializedBlueTableSchema() { }
 
         //copies the schema from an existing schema, building the dispatch table
-        public SerializedBlueTableSchema(IBlueTableSchema toCopy)
-        {
+        public SerializedBlueTableSchema(IBlueTableSchema toCopy) {
             int numCols = toCopy.columnInfo.Length;
             ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length];
-            foreach (ColumnInfo toCopyCI in toCopy.columnInfo)
-            {
+            foreach (ColumnInfo toCopyCI in toCopy.columnInfo) {
                 theColumns[toCopyCI.colNo] = toCopyCI;
             }
             myConstruct(theColumns, toCopy.tableName);
         }
 
-        public SerializedBlueTableSchema(DataTable table)
-        {
+        public SerializedBlueTableSchema(DataTable table) {
             ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count];
             myTableName = table.TableName;
-            foreach (DataColumn col in table.Columns)
-            {
+            foreach (DataColumn col in table.Columns) {
                 theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
             }
             myConstruct(theseCols, myTableName);
         }
 
-        public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName)
-        {
+        public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) {
             myConstruct(cis, tableName);
         }
 
-        public void serializeObject(object data, BinaryWriter bw, int colNo)
-        {
+        public void serializeObject(object data, BinaryWriter bw, int colNo) {
             mySerializeDispatchTable[colNo](data, bw);
 
         }
 
-        private void myConstruct(ColumnInfo[] cis, string tableName)
-        {
+        private void myConstruct(ColumnInfo[] cis, string tableName) {
             myIndexByColOrder = new ColumnInfo[cis.Length];
             myTableName = tableName;
-            foreach (ColumnInfo ci in cis)
-            {
+            foreach (ColumnInfo ci in cis) {
                 myIndexByColOrder[ci.colNo] = ci;
                 myIndexByColName[ci.colName] = ci;
             }
@@ -179,32 +163,25 @@ namespace BCG
 
         public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
 
-        public ColumnInfo this[string colName]
-        {
-            get
-            {
+        public ColumnInfo this[string colName] {
+            get {
                 object retVal = myIndexByColName[colName];
                 return retVal != null ? retVal as ColumnInfo : null;
             }
         }
 
-        public ColumnInfo this[int colNo]
-        {
+        public ColumnInfo this[int colNo] {
             get { return myIndexByColOrder[colNo]; }
         }
 
         /// <summary>
         /// The returned Array is sorted according to the Column No
         /// </summary>
-        public string[] columnNames
-        {
-            get
-            {
-                if (myTableColNames == null)
-                {
+        public string[] columnNames {
+            get {
+                if (myTableColNames == null) {
                     myTableColNames = new string[myIndexByColName.Count];
-                    foreach (DictionaryEntry de in myIndexByColName)
-                    {
+                    foreach (DictionaryEntry de in myIndexByColName) {
                         ColumnInfo temp = (ColumnInfo)de.Value;
                         myTableColNames[temp.colNo] = temp.colName;
                     }
@@ -216,64 +193,46 @@ namespace BCG
         /// <summary>
         /// The returned Array is sorted according to the Column No
         /// </summary>
-        public ColumnInfo[] columnInfo
-        {
+        public ColumnInfo[] columnInfo {
             get { return myIndexByColOrder; }
         }
 
         public string tableName { get { return myTableName; } }
 
 
-        public int IdColumn
-        {
+        public int IdColumn {
             get { return myIdCol; }
         }
 
 
         #region ISerializedType methods
-        public void serialize(BinaryWriter bw)
-        {
+        public void serialize(BinaryWriter bw) {
             bw.Write(columnNames.Length);
             bw.Write(tableName);
-            foreach (ColumnInfo ci in columnInfo)
-            {
+            foreach (ColumnInfo ci in columnInfo) {
                 bw.Write(ci.colName);
                 bw.Write(ci.colType.ToString());
-                if (ci.isConstant)
-                {
-                    bw.Write(true);
-                    SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
-                }
-                else
-                {
-                    bw.Write(false);
-                }
+                bw.Write(false);
             }
         }
 
-        public void deserialze(BinaryReader br)
-        {
+        public void deserialze(BinaryReader br) {
             ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
             string name = br.ReadString();
-            for (int i = 0; i < cis.Length; i++)
-            {
+            for (int i = 0; i < cis.Length; i++) {
                 string colName = br.ReadString();
                 Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
                 bool isConstant = br.ReadBoolean();
-                if (isConstant)
-                {
-                    cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t));
-                }
-                else
-                {
+                if (isConstant) {
+                    cis[i] = new ColumnInfo(colName, i, t);
+                } else {
                     cis[i] = new ColumnInfo(colName, i, t);
                 }
             }
             myConstruct(cis, name);
         }
 
-        public int getSerializedSize()
-        {
+        public int getSerializedSize() {
             throw new NotImplementedException("Feature for provided constant values to" +
                "non-primitive types in schema, not provided as of now");
         }
@@ -282,8 +241,7 @@ namespace BCG
 
 
         //builds a dispatch table used to pull values out of the memory streams in the rows. 
-        private void myBuildDispatchTable()
-        {
+        private void myBuildDispatchTable() {
             int numCols = myIndexByColOrder.Length;
             mySerializeDispatchTable = new SerializeDelegate[numCols];
             myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols];
@@ -292,18 +250,14 @@ namespace BCG
 
             myIdCol = idColumn.colNo;
 
-            for (int col = 0; col < numCols; ++col)
-            {
+            for (int col = 0; col < numCols; ++col) {
                 Type thisColType = myIndexByColOrder[col].colType;
                 //we don't store serializers for the id column, or string columns.
-                if (col == myIdCol || thisColType == typeof(string))
-                {
+                if (col == myIdCol || thisColType == typeof(string)) {
                     mySerializeDispatchTable[col] = null;
                     myColumnSizesTable[col] = 0;
                     myPullFromByteBufferDispatchTable[col] = null;
-                }
-                else
-                {
+                } else {
                     mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType);
                     myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType);
                     myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType);
@@ -315,16 +269,14 @@ namespace BCG
         }
 
 
-        public int getNullMapSize()
-        {
+        public int getNullMapSize() {
             int numCols = myIndexByColOrder.Length;
             return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0)));
         }
 
         //this function reads the null array from inside the byte array.
         //it has to copy the bytes over. 
-        public BitArray getNullMap(byte[] theBytes)
-        {
+        public BitArray getNullMap(byte[] theBytes) {
             int nullMapSize = getNullMapSize();
             //the null map starts after the id, so that we have instant access to the id. 
             int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32));
@@ -336,8 +288,7 @@ namespace BCG
         }
         //gets the value either from the bytes stream but only if the value is not a string. 
         //doesn't work if you're getting the id.
-        public unsafe object getValueNonString(byte[] theBytes, int colNo)
-        {
+        public unsafe object getValueNonString(byte[] theBytes, int colNo) {
             if (colNo == myIdCol)
                 throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row");
             BitArray nullMap = getNullMap(theBytes);
@@ -345,8 +296,7 @@ namespace BCG
                 return DBNull.Value;
             //the rest of the row starts after the id and the null map
             int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
-            for (int currentColumn = 0; currentColumn < colNo; ++currentColumn)
-            {
+            for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) {
                 //pull the data from the binary reader, but only if this type is not null or a string
                 //if the type wer'e looking for is a string, it would have been returned already by the accesor
                 //in the bluerow. 
@@ -357,16 +307,14 @@ namespace BCG
 
                     thisOffset += myColumnSizesTable[currentColumn];
             }
-            fixed (byte* thisBytePtr = &theBytes[thisOffset])
-            {
+            fixed (byte* thisBytePtr = &theBytes[thisOffset]) {
                 return myPullFromByteBufferDispatchTable[colNo](thisBytePtr);
             }
 
 
         }
 
-        public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray)
-        {
+        public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) {
             int numCols = myColumnSizesTable.Length;
             object[] theObjs = new Object[numCols];
 
@@ -374,8 +322,7 @@ namespace BCG
 
             int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
             int thisStringNo = 0;
-            for (int currentColumn = 0; currentColumn < numCols; ++currentColumn)
-            {
+            for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) {
                 //pull the data from the binary reader, but only if this type is not null or a string
                 //if the type wer'e looking for is a string, it would have been returned already by the accesor
                 //in the bluerow. 
@@ -384,10 +331,8 @@ namespace BCG
                     theObjs[currentColumn] = DBNull.Value;
                 else if (myIndexByColOrder[currentColumn].colType == typeof(string))
                     theObjs[currentColumn] = stringArray[thisStringNo++];
-                else if (currentColumn != myIdCol)
-                {
-                    fixed (byte* thisGuy = &theBytes[thisOffset])
-                    {
+                else if (currentColumn != myIdCol) {
+                    fixed (byte* thisGuy = &theBytes[thisOffset]) {
                         theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy);
                     }
                     thisOffset += myColumnSizesTable[currentColumn];
@@ -416,8 +361,7 @@ namespace BCG
         private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); }
         private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); }
 
-        private SerializeDelegate myGetDispatchMethodSerialize(Type t)
-        {
+        private SerializeDelegate myGetDispatchMethodSerialize(Type t) {
             if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32);
             else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble);
             else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar);
@@ -448,16 +392,14 @@ namespace BCG
         private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); }
         private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); }
         private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); }
-        private unsafe object myPullDateTime(byte* bytePtr)
-        {
+        private unsafe object myPullDateTime(byte* bytePtr) {
             double thisDateTime = *((double*)bytePtr);
             return DateTime.FromOADate(thisDateTime);
         }
         private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; }
 
 
-        private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t)
-        {
+        private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) {
             if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32);
             else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble);
             else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar);

src/DataStreamSuite/DataStreamCore/LightTableStream.cs

@@ -1,51 +1,44 @@
-using System;
-using System.Collections;
-using System.Data;
+// using System;
+// using System.Collections;
+// using System.Data;
 
-using BCG.Utilities;
+// using BCG.Utilities;
 
 
-/* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired.
- * This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains. 
- * 
- */
-namespace BCG
-{
-    public class LightTableStream : TableStream
-    {
-        public LightTableStream(string tableName)
-         :
-         base(tableName, null, null)
-      {
-      }
+// /* this class reprsents a table stream for use with CppDataStreams. It does not hold on to the last value fired.
+//  * This way, the last value is purged from managed memory and only the slimmer, sexier Cpp implementation of the row remains. 
+//  * 
+//  */
+// namespace BCG {
+//     public class LightTableStream : TableStream {
+//         public LightTableStream(string tableName)
+//          :
+//          base(tableName, null, null) {
+//         }
 
-      public LightTableStream(string tableName, ViewStreamDatabase db)
-         :
-         base(tableName, db, null)
-      {}
+//         public LightTableStream(string tableName, ViewStreamDatabase db)
+//            :
+//            base(tableName, db, null) { }
 
-      public LightTableStream(string tableName, string colExp)
-         :
-         base(tableName, 
-         null, 
-         colExp)
-      {}
-
-      
-      public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) :
-          base(ttableName,db,colExp)
-      {
-       }
+//         public LightTableStream(string tableName, string colExp)
+//            :
+//            base(tableName,
+//            null,
+//            colExp) { }
 
 
-        protected override void myHandleUpdates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
-        {
+//         public LightTableStream(string ttableName, ViewStreamDatabase db, string colExp) :
+//             base(ttableName, db, colExp) {
+//         }
 
-            base.myHandleUpdates(newRows, addedRows, deletedRows);
-            //once you've fired the rows, delete them becuase we don't want to hold onto them in memory
-            myLastValue = null;
-            
-        }
 
-    }
-}
+//         protected override void _handle_updates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) {
+
+//             base._handle_updates(newRows, addedRows, deletedRows);
+//             //once you've fired the rows, delete them becuase we don't want to hold onto them in memory
+//             myLastValue = null;
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs

@@ -1,351 +0,0 @@
-using System;
-using System.IO;
-//using BCG.Utilities;
-using mbts;
-using mbts.spark;
-
-namespace BCG
-{   
-	public class TableAccumulator : ITableAccumulator
-	{
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
-      private IClientDeserializer myDeserializer = new ClientDeserializationController ();
-      private BlueTable myStateTable;
-      private ISubscriberCore mySubscriptionCore;
-      private TableUpdateDelegate myOnUpdateDelegate;
-      private string myTag;
-      private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps;
-
-      public TableAccumulator (string tag, ISubscriberCore subCore) 
-      {
-         myTag = tag;
-         mySubscriptionCore = subCore;
-      }
-
-      #region ITableAccumulator Members
-
-      public bool isActive()
-      {return myOnUpdateDelegate != null;}
-
-      public event TableUpdateDelegate onUpdate
-      {
-         add
-         {
-            // BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information);
-            if (myOnUpdateDelegate == null)
-            {
-               //ie first subscription
-               // BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
-               mySubscriptionCore.subscribe (myTag);
-            }
-            myOnUpdateDelegate = 
-               Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
-            if (myStateTable != null)
-               value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
-      }
-         remove
-         {
-            // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
-            if (myOnUpdateDelegate != null)
-            {
-               myOnUpdateDelegate = 
-                  Delegate.Remove (myOnUpdateDelegate, value) as TableUpdateDelegate;
-               if (myOnUpdateDelegate == null)
-               { 
-                  //all subscribers gone
-                  // BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
-                  mySubscriptionCore.unsubscribe (myTag);
-                  myStateTable = null;                  
-               }
-            }
-
-         }
-      }
-
-      public void update(Stream[] streams)
-      {
-         bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
-         //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
-         //data element for that tag in the ViewStreamDatabase
-         if (myOnUpdateDelegate != null)
-         {
-            byte[] ops;
-            Array rows;
-            _timer.start();
-            myDeserializer.deserialize (streams, out ops, out rows);
-            _timer.stop();
-            // BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
-            _timer.start();
-            myAccumulateAndFire (ops, rows);
-            _timer.stop();
-            // BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
-         }
-      }
-
-      #endregion
-
-      private void myAccumulateAndFire (byte[] ops, Array rows)
-      {
-         //if a non create comes before the create, the deserializer returns 0 sized arrays
-         if (ops.Length == 0 || rows.Length == 0)
-         {
-            //ie if one is zero sized so should be the other one
-            // BlueEventLog.logAssert (ops.Length + rows.Length == 0);
-         }
-         else
-         {
-            if (ops.Length > 1)
-            {
-               int index = 0;
-               if (myStateTable == null)
-               {
-                  for(index=0; index<ops.Length; index++)
-                     if (ops[index] == StreamOpCode.create)
-                        break;
-                  if (index == ops.Length)
-                     return;
-               }
-               if (myUseIncrementsForMultipleOps)
-                  myHandleMultipleOpsIncrementally(ops, rows, index);
-               else
-                  myHandleMultipleOps(ops, rows, index);
-            }
-            else
-            {
-               if (myStateTable != null || ops[0] == StreamOpCode.create)
-                  myHandleSingleOp (ops, rows);
-            }
-         }
-      }
-
-      private void myHandleMultipleOps (byte[] ops, Array rows, int offset)
-      {
-         HighPerformanceTimer timer = new HighPerformanceTimer();
-         timer.start();
-         IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable (myStateTable);
-         for (int i = offset; i < ops.Length; i++)
-         {
-            byte op = ops[i];
-            IBlueRowCollection[] tableArray = rows.GetValue (i) as IBlueRowCollection[];
-            if (op == StreamOpCode.add)
-               myStateTable.addRows (tableArray[0]);
-            else if (op == StreamOpCode.remove)
-               myStateTable.deleteRows (tableArray[0]);
-            else if (op == StreamOpCode.change)
-               myStateTable.changeRows (tableArray[0]);
-            else if (op == StreamOpCode.create)
-               myStateTable = new BlueTable (tableArray[0]);
-            else if (op == StreamOpCode.multipleChange)
-               myStateTable.changeRows (tableArray[0]);
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-               myStateTable.deleteRows (tableArray[0]);
-               myStateTable.addRows (tableArray[1]);
-            }
-         }
-         IBlueRowCollection dRows, aRows;
-         if (stateAtStart == null)
-         {
-            dRows = new BlueTable (myStateTable.schema);
-            aRows = myStateTable;
-         }
-         else
-         {
-            BlueDataUtility.computeDifferences (myStateTable, stateAtStart, out dRows, out aRows);
-         }
-         timer.stop();
-         //Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs");
-         myOnUpdateDelegate (myStateTable, aRows, dRows);
-
-      }
-
-      #region handling mutliple ops in single update call, incrementally
-
-      private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset)
-      {
-         BlueTable dels = null, adds = null;
-         if (myStateTable != null)
-         {
-            dels = new BlueTable(myStateTable.schema);
-            adds = new BlueTable(myStateTable.schema);
-         }
-         for (int i = offset; i < ops.Length; i++)
-         {
-            byte op = ops[i];
-            IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
-            if (op == StreamOpCode.add)
-            {
-               myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
-            }
-            else if (op == StreamOpCode.remove)
-            {
-               myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
-            }
-            else if (op == StreamOpCode.change)
-            {
-               myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
-            }
-            else if (op == StreamOpCode.create)
-            {
-               myHandleCreate(tableArray[0], ref dels, ref adds);
-            }
-            else if (op == StreamOpCode.multipleChange)
-            {
-               myHandleMultipleChanges(tableArray[0], dels, adds);
-            }
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-               myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds);
-            }
-         }
-         myOnUpdateDelegate(myStateTable, adds, dels);
-      }
-
-      private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            myStateTable.addRow(r);
-            adds.addRow(r);
-         }
-      }
-
-      private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            myStateTable.deleteRow(r);
-            if (adds.containsID((int)r["id"]))
-               adds.deleteRow(r);
-            else
-               dels.addRow(r);
-         }
-      }
-
-      private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            IBlueRow er = myStateTable[(int)r["id"]];
-            myHandleRemove(er, dels, adds);
-            myHandleAdd(r, dels, adds);
-         }
-      }
-
-      private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds)
-      {
-         if (dels == null)
-         {
-            // BlueEventLog.logAssert(adds == null);
-            dels = new BlueTable(newTable.schema);
-            adds = new BlueTable(newTable);
-            myStateTable = new BlueTable(newTable);
-         }
-         else
-         {
-            dels = new BlueTable(myStateTable);
-            myStateTable = new BlueTable(newTable);
-            adds = new BlueTable(newTable);
-         }
-      }
-
-      private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows,
-                                             BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-         {
-            foreach (IBlueRow r in deletedRows)
-               myHandleRemove(r, dels, adds);
-            foreach (IBlueRow r in addedRows)
-               myHandleAdd(r, dels, adds);
-         }
-      }
-
-      private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds)
-      {
-         if (dels != null)
-            foreach (IBlueRow r in changedRows)
-               myHandleChange(r, dels, adds);
-      }
-
-      #endregion
-
-      private void myHandleSingleOp(byte[] ops, Array rows)
-      {
-         byte op = ops[0];
-         IBlueRowCollection[] tableArray = rows.GetValue (0) as IBlueRowCollection[];
-         IBlueRowCollection dRows, aRows;
-         if (op == StreamOpCode.create)
-         {
-            if (myStateTable == null)
-            {
-               aRows = myStateTable = new BlueTable(tableArray[0]);
-               dRows = new BlueTable(myStateTable.schema);
-               myOnUpdateDelegate(myStateTable, aRows, dRows);
-            }
-            else
-            {
-               IBlueRowCollection newState = new BlueTable(tableArray[0]);
-               BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows);
-               if (aRows.Count > 0 || dRows.Count > 0)
-               {
-                  myStateTable = new BlueTable(newState);
-                  myOnUpdateDelegate(myStateTable, aRows, dRows);
-               }
-            }
-         }
-         else
-         {
-            if (op == StreamOpCode.add)
-            {
-               myStateTable.addRows(aRows = tableArray[0]);
-               dRows = new BlueTable(myStateTable.schema);
-            }
-            else if (op == StreamOpCode.remove)
-            {
-               myStateTable.deleteRows(dRows = tableArray[0]);
-               aRows = new BlueTable(myStateTable.schema);
-            }
-            else if (op == StreamOpCode.change)
-            {
-               IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0);
-               aRows = new BlueTable(myStateTable.schema);
-               dRows = new BlueTable(myStateTable.schema);
-               (aRows as BlueTable).addRow(cRow);
-               (dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]);
-               myStateTable.changeRow(cRow);
-            }
-            else if (op == StreamOpCode.multipleChange)
-            {
-               if (myStateTable.Count == tableArray[0].Count)
-               {
-                  //all rows are changed, so we can optimize this
-                  dRows = myStateTable;
-                  aRows = myStateTable = new BlueTable(tableArray[0]);
-               }
-               else
-               {
-                  dRows = new BlueTable(myStateTable.schema);
-                  aRows = tableArray[0];
-                  foreach (IBlueRow r in tableArray[0])
-                  {
-                     (dRows as BlueTable).addRow(myStateTable[(int)r["id"]]);
-                     myStateTable.changeRow(r);
-                  }
-               }
-            }
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-               BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]);
-            }
-            else
-            {
-               aRows = dRows = null;
-               // BlueEventLog.logAssert(false, "unknown opcode");
-            }
-            myOnUpdateDelegate(myStateTable, aRows, dRows);
-         }
-      }
-   }
-   
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs

@@ -1,67 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-using BCG.Utilities;
-
-namespace BCG {
-   /// <summary>
-   /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
-   /// </summary>
-   public class ClientDeserializationController : IClientDeserializer {
-      private IClientDeserializer myDeserializer;
-      private int _n_desers = 0;
-
-      public ClientDeserializationController() { }
-
-      #region IClientDeserializer Members
-
-      /// <summary>
-      /// Deserializes bytes into operation and data. If it gets a non create before a create, both
-      /// the out arrays get initialized to empty arrays
-      /// </summary>
-      /// <param name="streams"></param>
-      /// <param name="theOps"></param>
-      /// <param name="theRows"></param>
-      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
-         bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
-         if (myDeserializer != null) {
-            myDeserializer.deserialize(streams, out theOps, out theRows);
-         } else {
-            Stream totalStream = UtilityMethods.getAggregatedStream(streams);
-            long initialPos = totalStream.Position;
-            BinaryReader br = new BinaryReader(totalStream);
-            byte op;
-            try {
-               op = br.ReadByte();
-            } catch (EndOfStreamException) {
-               op = Byte.MaxValue;
-            }
-            if (op == StreamOpCode.create) {
-               br.ReadString(); //pubsubclientdeserializer
-               string deserName = br.ReadString();
-               myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
-                  as IClientDeserializer;
-               totalStream.Position = initialPos;
-               myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
-            } else {
-               // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
-               theOps = new byte[] { };
-               theRows = new object[] { };
-            }
-         }
-         var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
-         int i = 0;
-         foreach (var m in msgs) {
-            bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
-         }
-         _n_desers++;
-         // log it here
-
-      }
-
-
-      #endregion
-
-   }
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs

@@ -1,124 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-using BCG.Utilities;
-//using BCG.Utilities.BlueConfiguration;
-
-namespace BCG
-{
-   /// <summary>
-   /// Pairs up with the UniversalSerializer to deserialize the corresponding data. Everythign anologous
-   /// to UniversalSerializer.
-   /// </summary>
-   public abstract class ClientDeserializer : IClientDeserializer
-   {
-      protected IBlueTableSchema mySchema;
-      protected IBlueTableSchema myFirstSchema;
-      protected IClientRowDeserializer myRowDeserializer;
-
-      public ClientDeserializer()
-      {
-         myFirstSchema = null;
-         mySchema = null;
-
-      }
-
-      #region IClientDeserializer Members
-
-      void BCG.IClientDeserializer.deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
-
-         // BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length);
-
-         // emit # of streams and total length
-         ArrayList ops = new ArrayList(), rows = new ArrayList();
-         IBlueRowCollection[] result;
-         while (true)
-         {
-            byte op;
-            try
-            {
-               int value = br.BaseStream.ReadByte();
-               if (value == -1) break;
-               op = (byte)value;
-
-               // BlueTrace.WriteDeserializeOperation(op);
-            }
-            catch (EndOfStreamException)
-            {
-               break;
-            }
-            if (op == StreamOpCode.create)
-            {
-
-               br.ReadString(); //pubsubDeserializer
-               br.ReadString(); //clientDeserializer
-               mySchema = new BlueTableSchema();
-               mySchema.deserialze(br);
-               //keep track of the first schema you've ever seen, so you know how to reorder columns
-               if (myFirstSchema == null)
-                  myFirstSchema = mySchema;
-
-               result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) };
-               myEncorporateRows(result[0], op);
-               ops.Add(op);
-               rows.Add(result);
-            }
-            else
-            {
-               if (mySchema != null)
-               {
-                  if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
-                  {
-                     result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
-                     myEncorporateRows(result[0], op);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else if (op == StreamOpCode.multipleChange)
-                  {
-                     result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) };
-                     myEncorporateRows(result[0], op);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else if (op == StreamOpCode.multipleRemoveAdd)
-                  {
-                     result =
-                        new IBlueRowCollection[] {
-                                                     myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()),
-                                                     myGetRows (br, StreamOpCode.add, br.ReadInt32 ())
-                                                  };
-
-                     myEncorporateRows(result[0], StreamOpCode.remove);
-                     myEncorporateRows(result[1], StreamOpCode.add);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else
-                  {
-                     // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
-                  }
-               }
-               else
-               {
-                  // BlueEventLog.logEvent("Received a non create operation even though the schema" +
-                  //    "has never been created", BlueEventLevel.Warning);
-               }
-            }
-         }
-         theOps = ops.ToArray(typeof(byte)) as byte[];
-         theRows = rows.ToArray();
-         // BlueTrace.WriteDeserializeStop(rows.Count);
-      }
-
-      #endregion
-
-      protected abstract IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows);
-
-      protected abstract void myEncorporateRows(IBlueRowCollection rows, byte op);
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs

@@ -1,34 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-
-namespace BCG
-{
-   class ClientReorderingRowDeserializer : IClientRowDeserializer
-   {
-      private IBlueTableSchema myOriginalSchema;
-      private IBlueTableSchema myOutputSchema;
-      private ClientUniversalRowDeserializer myDeserializer;
-      public ClientReorderingRowDeserializer(IBlueTableSchema originalSchema, IBlueTableSchema newSchema)
-      {
-         myOriginalSchema = originalSchema;
-         myOutputSchema = newSchema;
-         myDeserializer = new ClientUniversalRowDeserializer(newSchema);
-         
-      }
-
-      #region IClientRowDeserializer Members
-
-      public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
-      {
-         IBlueRow temp = null;
-         myDeserializer.deserializeRow(currentState, br, op, out temp);
-         r = BlueDataUtility.changeSchema(temp, myOutputSchema);
-
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs

@@ -1,35 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-namespace BCG
-{
-   public class ClientUniversalDeserializer : ClientDeserializer
-   {
-      public ClientUniversalDeserializer ()  {}
-
-      protected override IBlueRowCollection myGetRows (BinaryReader br, byte op, int nRows)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
-
-         if (myRowDeserializer == null)
-            myRowDeserializer = new ClientUniversalRowDeserializer(mySchema);
-         else if (op == StreamOpCode.create)
-            myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema);
-         
-         for (int i = 0; i < nRows; i++)
-         {
-            IBlueRow row;
-            myRowDeserializer.deserializeRow (null, br, op, out row);
-            retVal.addRow (row);
-         }
-         return retVal;
-      }
-
-      protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
-      {
-         //this does not keep state so ignore calls
-      }
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs

@@ -1,69 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-using BCG.Utilities;
-//using BCG.Utilities.BlueConfiguration;
-
-namespace BCG {
-   /// <summary>
-   /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
-   /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) 
-   /// </summary>
-   internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
-      private IBlueTableSchema mySchema;
-      private int myNullMapLength = 0;
-      private PrimitiveDeserializationHelper myDeserializationHelper;
-      private BitArray myNullMap;
-      private int[] mySerIndexToSchemaIndexMap;
-
-      public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
-         myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
-         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
-         myInitReorderSchema();
-
-
-      }
-
-      #region Schema Reordering initialization
-      private void myInitReorderSchema() {
-         try {
-            int idColIndex = mySchema["id"].colNo;
-            int[] temp = new int[mySchema.columnInfo.Length];
-            foreach (ColumnInfo ci in mySchema.columnInfo) {
-               temp[ci.colNo] =
-                  ci.colNo == idColIndex ? 0 :
-                  ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
-            }
-            mySerIndexToSchemaIndexMap = new int[temp.Length];
-            for (int i = 0; i < temp.Length; i++)
-               mySerIndexToSchemaIndexMap[temp[i]] = i;
-         } catch (Exception ex) {
-            string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
-            string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
-            bts.Log.Gen.error(err_msg);
-            throw new ApplicationException(err_msg);
-         }
-      }
-      #endregion
-
-      #region IClientRowDeserializer Members
-
-      public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
-         br.ReadInt32();  //length information for the row not needed
-         myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
-         object[] values = new object[mySchema.columnNames.Length];
-         for (int i = 0; i < values.Length; i++) {
-            int colNo = mySerIndexToSchemaIndexMap[i];
-            values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
-         }
-
-
-         r = new BlueRow(mySchema, values);
-
-      }
-
-      #endregion
-   }
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs

@@ -1,149 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-
-namespace BCG {
-   public class StreamIOMessage {
-      public enum StreamOp {
-         Create = 0,
-         Add = 1,             //adds a single row
-         Remove = 2,          //removes a single row
-         Change = 3,          //changes a single row
-         MultipleRemoveAdd = 4,//removes and adds multiple rows
-         MultipleChange = 5 //changes multiples rows          
-      }
-      public class CreateMessage {
-         public IBlueRowCollection rows;
-         public override string ToString() {
-            string rv = "Create message ";
-            if (rows == null) {
-               rv += " rows: null";
-               return rv;
-            }
-            foreach (IBlueRow r in rows) {
-               rv += $"id: {r["id"]} ";
-            }
-            return rv;
-         }
-      }
-      public class AddMessage {
-         public IBlueRow added_row;
-      }
-      public class RemoveMessage {
-         public IBlueRow removed_row;
-      }
-      public class ChangeMessage {
-         public IBlueRow changed_row;
-      }
-      public class MultipleRemoveAddMessage {
-         public IBlueRowCollection removed;
-         public IBlueRowCollection added;
-         public override string ToString() {
-            string rv = "MultipleRemoveAdd removed: ";
-            foreach (IBlueRow r in removed) {
-               rv += $"id: {r["id"]} ";
-            }
-            rv += "added: ";
-            foreach (IBlueRow r in added) {
-               rv += $"id: {r["id"]} ";
-            }
-            return rv;
-         }
-      }
-      public class MultipleChangeMessage {
-         public IBlueRowCollection changed;
-      }
-
-      public StreamOp op;
-      public CreateMessage create_message = null;
-      public AddMessage add_message = null;
-      public RemoveMessage remove_message = null;
-      public ChangeMessage change_message = null;
-      public MultipleRemoveAddMessage multiple_remove_add_message = null;
-      public MultipleChangeMessage multiple_change_message = null;
-      public StreamIOMessage(byte op_, object data) {
-         op = (StreamIOMessage.StreamOp)(op_);
-         switch (op) {
-            case StreamIOMessage.StreamOp.Create:
-               IBlueRowCollection state = null;
-               // TODO, not sure why it can be an array and by itself
-               if (data is IBlueRowCollection) state = data as IBlueRowCollection;
-               else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
-               create_message = new CreateMessage() { rows = state };
-               break;
-            case StreamIOMessage.StreamOp.Add:
-               bts.Log.assert(false);
-               break;
-            case StreamIOMessage.StreamOp.Remove:
-               bts.Log.assert(false);
-               break;
-            case StreamIOMessage.StreamOp.Change:
-               bts.Log.assert(false);
-               break;
-            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
-               IBlueRowCollection[] row_args = data as IBlueRowCollection[];
-               multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
-               break;
-            case StreamIOMessage.StreamOp.MultipleChange:
-               //IBlueRowCollection[] row_args = data as IBlueRowCollection[];
-               multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
-               break;
-         }
-      }
-
-
-      public override string ToString() {
-         switch (op) {
-            case StreamIOMessage.StreamOp.Create:
-               return create_message.ToString();
-               break;
-            case StreamIOMessage.StreamOp.Add:
-               bts.Log.assert(false);
-               return String.Empty;
-               break;
-            case StreamIOMessage.StreamOp.Remove:
-               bts.Log.assert(false);
-               return String.Empty;
-               break;
-            case StreamIOMessage.StreamOp.Change:
-               bts.Log.assert(false);
-               return String.Empty;
-               break;
-            case StreamIOMessage.StreamOp.MultipleRemoveAdd:
-               return multiple_remove_add_message.ToString();
-               break;
-            case StreamIOMessage.StreamOp.MultipleChange:
-               return multiple_change_message.ToString();
-               break;
-         }
-         bts.Log.assert(false);
-         return String.Empty;
-      }
-      public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
-         int i = 0;
-         List<StreamIOMessage> rv = new List<StreamIOMessage>();
-         foreach (byte op in ops) {
-            object data = op_data.GetValue(i++);
-            rv.Add(new StreamIOMessage(op, data));
-         }
-         return rv;
-      }
-   }
-
-   /// <summary>
-   /// Performs deserialization inside the client app. The client gets either the complete data set or the
-   /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
-   /// </summary>
-   public interface IClientDeserializer {
-      /// <summary>
-      /// deserializes the data received by the client from the pubsub
-      /// </summary>
-      /// <param name="serializedData">the data in byte format received from pubsub</param>
-      /// <param name="theOps">the operations that the data maps onto</param>
-      /// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
-      void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
-   }
-}
-
-

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs

@@ -1,12 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-
-namespace BCG {
-   /// <summary>
-   /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. 
-   /// </summary>
-   public interface IClientRowDeserializer {
-      void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs

@@ -1,79 +0,0 @@
-using System;
-using System.IO;
-
-namespace BCG
-{
-   /// <summary>
-   /// Deserializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1)
-   /// rather than searching it up and making it O(n)
-   /// </summary>
-	public class PrimitiveDeserializationHelper
-	{
-      private delegate object DeserializeDelegate (int colNo, BinaryReader br);
-      private DeserializeDelegate[] myDispatchTable;
-      private ColumnInfo[] myCis;
-
-      public PrimitiveDeserializationHelper (IBlueTableSchema schema)
-         : this (schema.columnInfo)
-      {}
-
-		public PrimitiveDeserializationHelper (ColumnInfo[] cis)
-		{
-         myDispatchTable = new DeserializeDelegate [(myCis = cis).Length];
-         foreach (ColumnInfo ci in cis)
-            myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType);
-		}
-
-      public object deserializeColumn (int colNo, BinaryReader br)
-      {
-         return myDispatchTable[colNo] (colNo, br);
-      }
-
-      private DeserializeDelegate myGetDispatchMethod (Type t)
-      {
-         if (t.Equals (BlueDataValidTypes.INT)) return new DeserializeDelegate (myDeserializeInt32);
-         else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new DeserializeDelegate (myDeserializeDouble);
-         else if (t.Equals (BlueDataValidTypes.STRING)) return new DeserializeDelegate (myDeserializeString);
-         else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new DeserializeDelegate (myDeserializeByteArray);
-         else if (t.Equals (BlueDataValidTypes.CHAR)) return new DeserializeDelegate (myDeserializeChar);
-         else if (t.Equals (BlueDataValidTypes.LONG)) return new DeserializeDelegate (myDeserializeLong);
-         else if (t.Equals (BlueDataValidTypes.SBYTE)) return new DeserializeDelegate (myDeserializeSByte);
-         else if (t.Equals (BlueDataValidTypes.BOOL)) return new DeserializeDelegate (myDeserializeBool);
-         else if (t.Equals (BlueDataValidTypes.BYTE)) return new DeserializeDelegate (myDeserializeByte);
-         else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new DeserializeDelegate (myDeserializeDecimal);
-         else if (t.Equals (BlueDataValidTypes.UINT)) return new DeserializeDelegate (myDeserializeUInt);
-         else if (t.Equals (BlueDataValidTypes.ULONG)) return new DeserializeDelegate (myDeserializeULong);
-         else if (t.Equals (BlueDataValidTypes.DATETIME)) return new DeserializeDelegate (myDeserializeDateTime);  
-         else if (t.Equals (BlueDataValidTypes.SINGLE)) return new DeserializeDelegate (myDeserializeSingle);
-         else if (SerializationUtilityMethods.isSerializedType (t)) return new DeserializeDelegate (myDeserializeISerializedType);
-         else throw new ApplicationException (t.FullName + " is not a supported type");
-      }
-
-      private object myDeserializeInt32 (int colNo, BinaryReader br) {return br.ReadInt32 ();}
-      private object myDeserializeISerializedType (int colNo, BinaryReader br)
-      {
-         ISerializedType o = Activator.CreateInstance (myCis[colNo].colType)
-            as ISerializedType;
-         o.deserialze (br);
-         return o;
-      }
-      private object myDeserializeDouble (int colNo, BinaryReader br) {return br.ReadDouble ();}
-      private object myDeserializeString (int colNo, BinaryReader br) {return br.ReadString ();}
-      private object myDeserializeByteArray (int colNo, BinaryReader br) {return br.ReadBytes (br.ReadInt32 ());}
-      private object myDeserializeChar (int colNo, BinaryReader br) {return br.ReadChar ();}
-      private object myDeserializeLong (int colNo, BinaryReader br) {return br.ReadInt64 ();}
-      private object myDeserializeSByte (int colNo, BinaryReader br) {return br.ReadSByte ();}
-      private object myDeserializeBool (int colNo, BinaryReader br) {return br.ReadBoolean ();}
-      private object myDeserializeByte (int colNo, BinaryReader br) {return br.ReadByte ();}
-      private object myDeserializeDecimal (int colNo, BinaryReader br) {return br.ReadDecimal ();}
-      private object myDeserializeUInt (int colNo, BinaryReader br) {return br.ReadUInt32 ();}
-      private object myDeserializeDateTime (int colNo, BinaryReader br)
-      {
-         return new DateTime(br.ReadInt64());
-         //return DateTime.FromOADate (br.ReadDouble ());
-         //return new DateTime(br.ReadInt64());
-      }
-      private object myDeserializeULong (int colNo, BinaryReader br) {return br.ReadUInt64 ();}
-      private object myDeserializeSingle (int colNo, BinaryReader br) {return br.ReadSingle ();}
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs

@@ -1,155 +0,0 @@
-using System;
-using System.Collections;
-using System.Reflection;
-//using BCG.Utilities.BlueConfiguration;
-using BCG.Utilities;
-
-namespace BCG
-{
-	public class CorePrimtiveManager
-	{
-      private IDictionary myTypeLookupTable = new Hashtable ();
-      private static CorePrimtiveManager mySelf;
-      private ArrayList myAssemblyNames = new ArrayList ();
-
-      private CorePrimtiveManager() 
-      {
-         myLoadAllTypes ();
-      }
-
-      private void myLoadAllTypes ()
-      {
-          //we only load one type here
-         loadAssembly ("X:\\bcg\\blue2\\Debug\\CppToCSharpDataStreamBridge.dll");
-      }
-
-      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
-      {
-         foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
-            if (args.Name.Equals(a.FullName))
-               return a;
-         return null;
-      }
-
-      /// <summary>
-      /// Loads all the types associated with the assembly
-      /// </summary>
-      /// <param name="assemblyName"></param>
-      public void loadAssembly (string assemblyName)
-      {
-         if (myAssemblyNames.Contains (assemblyName))
-            return; 
-         myAssemblyNames.Add (assemblyName);
-         Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ?
-            GetType ().Assembly :
-            Assembly.LoadFrom (assemblyName);
-
-         foreach (Type type in loadedAssembly.GetTypes())
-            if (!type.IsInterface
-               && !type.IsAbstract 
-               && type.IsPublic)
-            {
-               bool isVisible = true;
-               string userRep = type.Name;
-               
-               if (isVisible)
-               {
-                  // BlueEventLog.logAssert(myTypeLookupTable [userRep] == null);
-                  myTypeLookupTable[userRep] = type;
-               }
-            }
-      }
-
-
-      
-
-      public static CorePrimtiveManager instance
-      {
-         get {return mySelf = mySelf == null ? new CorePrimtiveManager () : mySelf;}
-      }
-
-      /// <summary>
-      /// Returns the Type name corresponding to the primitive name
-      /// </summary>
-      /// <param name="friendlyName"></param>
-      /// <returns></returns>
-      public string find (string friendlyName)  
-      {
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
-         return (myTypeLookupTable[friendlyName] as Type).Name;
-      }
-
-      /// <summary>
-      /// Constructs and returns an object of type corresponding to the friendly name
-      /// </summary>
-      /// <param name="friendlyName">friendly primitive name </param>
-      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
-      /// is the desired one</param>
-      /// <returns></returns>
-      public Object build (string friendlyName, ICollection args)   
-      {
-         object[] constructorArgs = null;
-         if (args != null && args.Count > 0)
-         {
-            constructorArgs = new object [args.Count];
-            args.CopyTo (constructorArgs, 0);
-         }
-         if (myTypeLookupTable[friendlyName] == null)
-            throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
-         return 
-            constructorArgs == null ?
-            Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type):
-            Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs);
-      }
-
-      /// <summary>
-      /// Returns the name of all the primitives 
-      /// </summary>
-      /// <returns></returns>
-      public string[] getPrimitiveNames ()
-      {
-         string[] retVal = new string [myTypeLookupTable.Count];
-         myTypeLookupTable.Keys.CopyTo (retVal, 0);
-         return retVal;
-      }
-
-      /// <summary>
-      /// Return the types of all the streams
-      /// </summary>
-      /// <returns></returns>
-      public Type[] getStreamTypes ()
-      {
-         Type[] retVal = new Type [myTypeLookupTable.Count];
-         myTypeLookupTable.Values.CopyTo (retVal, 0);
-         return retVal;
-      }
-
-      public ICollection getConstructorSigns (string primitiveName)
-      {
-         ArrayList retVal = new ArrayList ();
-         // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
-         Type t = myTypeLookupTable[primitiveName] as Type;
-         foreach (ConstructorInfo ci in t.GetConstructors())
-         {
-            string sign = "";
-            sign = t.Name + " (";
-            foreach (ParameterInfo pi in ci.GetParameters ())
-               sign += pi.ParameterType.Name + 
-                  " " + 
-                  pi.Name + ", ";
-            sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")";
-            retVal.Add (sign);
-         }
-         return retVal;
-      }
-
-      public string getAssemblyName (string primitiveName)
-      {
-         // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
-         return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
-
-      }
-
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs

@@ -1,20 +0,0 @@
-using System;
-
-namespace BCG
-{
-   public class PubStreamPrefixer
-   {
-      private static readonly string myPrefixString = "DS";
-      
-      public static string prefix (string tableName)
-      {
-         return myPrefixString + "_" + tableName.ToLower();
-      }
-
-      public static string extractTableNameFromTag (string tag)
-      {
-         return tag.Substring ((myPrefixString + "_").Length);
-      }
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs

@@ -1,339 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-
-namespace BCG
-{
-   public class SerializationUtilityMethods
-   {
-      private static IDictionary mySerializationDispatchTable,
-         myDeserializationDispatchTable;
-      private delegate void MySerialize (object data, BinaryWriter bw);
-      private delegate object MyDeserialize (Type t, BinaryReader br);
-
-      static SerializationUtilityMethods () 
-      {
-         mySerializationDispatchTable = new Hashtable ();
-         mySerializationDispatchTable [BlueDataValidTypes.BOOL.Name] = 
-            new MySerialize (mySerializeBool);
-         mySerializationDispatchTable [BlueDataValidTypes.BYTE.Name] = 
-            new MySerialize (mySerializeByte);
-         mySerializationDispatchTable [BlueDataValidTypes.CHAR.Name] = 
-            new MySerialize (mySerializeChar);
-         mySerializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = 
-            new MySerialize (mySerializeDateTime);
-         mySerializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = 
-            new MySerialize (mySerializeDecimal);
-         mySerializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = 
-            new MySerialize (mySerializeDouble);
-         mySerializationDispatchTable [BlueDataValidTypes.INT.Name] = 
-            new MySerialize (mySerializeInt32);
-         mySerializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = 
-            new MySerialize (mySerializeISerializedType);
-         mySerializationDispatchTable [BlueDataValidTypes.LONG.Name] = 
-            new MySerialize (mySerializeLong);
-         mySerializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = 
-            new MySerialize (mySerializeSByte);
-         mySerializationDispatchTable [BlueDataValidTypes.STRING.Name] = 
-            new MySerialize (mySerializeString);
-         mySerializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = 
-            new MySerialize (mySerializeByteArray);
-         mySerializationDispatchTable [BlueDataValidTypes.UINT.Name] = 
-            new MySerialize (mySerializeUInt);
-         mySerializationDispatchTable [BlueDataValidTypes.ULONG.Name] = 
-            new MySerialize (mySerializeULong);
-         mySerializationDispatchTable [BlueDataValidTypes.SINGLE.Name] =
-            new MySerialize (mySerializeSingle);
-
-         myDeserializationDispatchTable = new Hashtable ();
-         myDeserializationDispatchTable [BlueDataValidTypes.BOOL.Name] = 
-            new MyDeserialize (myDeserializeBool);
-         myDeserializationDispatchTable [BlueDataValidTypes.BYTE.Name] = 
-            new MyDeserialize (myDeserializeByte);
-         myDeserializationDispatchTable [BlueDataValidTypes.CHAR.Name] = 
-            new MyDeserialize (myDeserializeChar);
-         myDeserializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = 
-            new MyDeserialize (myDeserializeDateTime);
-         myDeserializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = 
-            new MyDeserialize (myDeserializeDecimal);
-         myDeserializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = 
-            new MyDeserialize (myDeserializeDouble);
-         myDeserializationDispatchTable [BlueDataValidTypes.INT.Name] = 
-            new MyDeserialize (myDeserializeInt32);
-         myDeserializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = 
-            new MyDeserialize (myDeserializeISerializedType);
-         myDeserializationDispatchTable [BlueDataValidTypes.LONG.Name] = 
-            new MyDeserialize (myDeserializeLong);
-         myDeserializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = 
-            new MyDeserialize (myDeserializeSByte);
-         myDeserializationDispatchTable [BlueDataValidTypes.STRING.Name] = 
-            new MyDeserialize (myDeserializeString);
-         myDeserializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = 
-            new MyDeserialize (myDeserializeByteArray);
-         myDeserializationDispatchTable [BlueDataValidTypes.UINT.Name] = 
-            new MyDeserialize (myDeserializeUInt);
-         myDeserializationDispatchTable [BlueDataValidTypes.ULONG.Name] = 
-            new MyDeserialize (myDeserializeULong);
-         myDeserializationDispatchTable [BlueDataValidTypes.SINGLE.Name] =
-            new MyDeserialize (myDeserializeSingle);
-      }
-
-      public static int getObjectSerializedSize (object o)
-      {
-         Type t = o.GetType ();
-         if (t == BlueDataValidTypes.STRING)
-         {
-            int l = (o as string).Length;
-            if (l < 128)
-               return l + 1;
-            if (l < 16384)
-               return l + 2;
-            if (l < 2097152)
-               return l + 3;
-            if (l < Math.Pow (2, 28))
-               return l + 4;
-            throw new ApplicationException ("String too big");
-         }
-         else if (t == BlueDataValidTypes.BYTEARRAY)
-         {
-            return getFixedLengthTypeSize (BlueDataValidTypes.INT) + 
-               (o as byte[]).Length;
-         }
-         else if (isSerializedType (t))
-         {
-            return (o as ISerializedType).getSerializedSize ();
-         }
-         return getFixedLengthTypeSize (t);
-      }
-
-      public static int getFixedLengthTypeSize (Type t)
-      {
-         if (t == BlueDataValidTypes.BOOL || 
-            t == BlueDataValidTypes.BYTE || 
-            t == BlueDataValidTypes.CHAR || 
-            t == BlueDataValidTypes.SBYTE)
-            return 1;
-         else if (t == BlueDataValidTypes.INT || 
-            t == BlueDataValidTypes.UINT||
-            t == BlueDataValidTypes.SINGLE)
-            return 4;
-         else if (t == BlueDataValidTypes.DOUBLE ||
-            t == BlueDataValidTypes.LONG ||
-            t== BlueDataValidTypes.ULONG)
-            return 8;
-         else if (t == BlueDataValidTypes.DECIMAL)
-            return 16;
-         else if (t == BlueDataValidTypes.DATETIME)
-            //as objects of type DateTime are stored as doubles
-            return getFixedLengthTypeSize (typeof(long));
-         else 
-            throw new ApplicationException (t.FullName + " is not a supported type");
-
-      }
-
-      public static bool isFixedLengthType (Type t)
-      {
-         return !(t.Equals (BlueDataValidTypes.STRING) || t.Equals (BlueDataValidTypes.BYTEARRAY) ||
-            isSerializedType (t));
-      }
-
-      public static bool isSerializedType (Type t)
-      {
-         return t.GetInterface ("BCG.ISerializedType") != null;
-      }
-
-      #region Serialization
-      public static void serializeValue (object data, BinaryWriter bw)
-      {
-         Type t = data.GetType ();
-         if (isSerializedType (t))
-            (mySerializationDispatchTable [typeof(ISerializedType).Name]
-               as MySerialize)(data, bw);
-         else
-            (mySerializationDispatchTable [t.Name] as MySerialize) (data, bw);
-      }
-      public static void serializeEnum(object data, BinaryWriter bw)
-      {
-         mySerializeInt32(data, bw);
-      }
-      private static void mySerializeInt32 (object data, BinaryWriter bw)
-      {
-         bw.Write ((Int32)data);
-      }
-
-      private static void mySerializeDouble (object data, BinaryWriter bw)
-      {
-         bw.Write ((double)data);
-      }
-
-      private static void mySerializeString (object data, BinaryWriter bw)
-      {
-         bw.Write (data as string);
-      }
-
-      private static void mySerializeByteArray (object data, BinaryWriter bw)
-      {
-         byte[] bs = data as byte[];
-         bw.Write (bs.Length);
-         bw.Write (bs);
-      }
-
-
-      private static void mySerializeSingle (object data, BinaryWriter bw)
-      {
-         bw.Write ((Single)data);
-      }
-
-      private static void mySerializeChar (object data, BinaryWriter bw)
-      {
-         bw.Write ((char)data);
-      }
-
-      private static void mySerializeLong (object data, BinaryWriter bw)
-      {
-         bw.Write ((long)data);
-      }
-
-      private static void mySerializeSByte (object data, BinaryWriter bw)
-      {
-         bw.Write ((sbyte)data);
-      }
-
-      private static void mySerializeBool (object data, BinaryWriter bw)
-      {
-         bw.Write ((bool)data);
-      }
-
-      private static void mySerializeByte (object data, BinaryWriter bw)
-      {
-         bw.Write ((byte)data);
-      }
-
-      private static void mySerializeDecimal (object data, BinaryWriter bw)
-      {
-         bw.Write ((decimal)data);
-      }
-
-      private static void mySerializeUInt (object data, BinaryWriter bw)
-      {
-         bw.Write ((uint)data);
-      }
-
-      private static void mySerializeULong (object data, BinaryWriter bw)
-      {
-         bw.Write ((ulong)data);
-      }
-
-      private static void mySerializeDateTime (object data, BinaryWriter bw)
-      {
-         //DateTime is written is serialiazed as OADATE
-         bw.Write (((DateTime)data).ToOADate());
-      }
-      
-      private static void mySerializeISerializedType (object data, BinaryWriter bw)
-      {
-         (data as ISerializedType).serialize (bw);
-      }
-      #endregion
-
-      #region Deserialization
-
-      public static object deserializeValue(Type t, BinaryReader br)
-      {
-         return myDeserialize(t, br);
-      }
-      private static object myDeserialize (Type t, BinaryReader br)
-      {
-         return isSerializedType (t) ?
-            (myDeserializationDispatchTable [typeof(ISerializedType).Name]
-            as MyDeserialize)(t, br)
-            :
-            (myDeserializationDispatchTable [t.Name] as MyDeserialize) (t, br);
-         }
-
-      public static object deserializeEnum(Type t, BinaryReader br)
-      {
-         return myDeserializeInt32(t, br); 
-      }
-
-      private static object myDeserializeInt32 (Type t, BinaryReader br)
-      {
-         return br.ReadInt32 ();
-      }
-
-      private static object myDeserializeISerializedType (Type t, BinaryReader br)
-      {
-         ISerializedType o = Activator.CreateInstance (t)
-            as ISerializedType;
-         o.deserialze (br);
-         return o;
-      }
-
-      private static object myDeserializeDouble (Type t, BinaryReader br)
-      {
-         return br.ReadDouble ();
-      }
-
-      private static object myDeserializeString (Type t, BinaryReader br)
-      {
-         return br.ReadString ();
-      }
-
-      private static object myDeserializeByteArray (Type t, BinaryReader br)
-      {
-         return br.ReadBytes (br.ReadInt32 ());
-      }
-
-      private static object myDeserializeSingle (Type t, BinaryReader br)
-      {
-         return br.ReadSingle ();
-      }
-
-      private static object myDeserializeChar (Type t, BinaryReader br)
-      {
-         return br.ReadChar ();
-      }
-
-      private static object myDeserializeLong (Type t, BinaryReader br)
-      {
-         return br.ReadInt64 ();
-      }
-
-      private static object myDeserializeSByte (Type t, BinaryReader br)
-      {
-         return br.ReadSByte ();
-      }
-
-      private static object myDeserializeBool (Type t, BinaryReader br)
-      {
-         return br.ReadBoolean ();
-      }
-
-      private static object myDeserializeByte (Type t, BinaryReader br)
-      {
-         return br.ReadByte ();
-      }
-
-      private static object myDeserializeDecimal (Type t, BinaryReader br)
-      {
-         return br.ReadDecimal ();
-      }
-
-      private static object myDeserializeUInt (Type t, BinaryReader br)
-      {
-         return br.ReadUInt32 ();
-      }
-
-      private static object myDeserializeDateTime (Type t, BinaryReader br)
-      {
-         return DateTime.FromOADate(br.ReadDouble());
-         //return new DateTime(br.ReadInt64());
-      }
-
-      private static object myDeserializeULong (Type t, BinaryReader br)
-      {
-         return br.ReadUInt64 ();
-      }
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs

@@ -1,44 +0,0 @@
-using System;
-
-namespace BCG
-{
-	/// <summary>
-	/// The table publishing happens here
-	/// </summary>
-	public interface IPubTablePublisher
-	{
-      /// <summary>
-      /// Adds a row to the existing table
-      /// </summary>
-      /// <param name="row"></param>
-      void addRow (IBlueRow row);
-      /// <summary>
-      /// Deletes a row from an existing table
-      /// </summary>
-      /// <param name="row"></param>
-      void deleteRow (IBlueRow row);
-      /// <summary>
-      /// Changes an existing row
-      /// </summary>
-      /// <param name="row"></param>
-      void changeRow (IBlueRow row);
-      /// <summary>
-      /// Creates a table. if already exists overwrites it.
-      /// </summary>
-      /// <param name="rows"></param>
-      void createTable (IBlueRowCollection rows);
-      /// <summary>
-      /// Removes and adds more than one row; however the assumption here is that delete is
-      /// applied first and then add. If that violates the current state, exception is thrown
-      /// </summary>
-      /// <param name="delRows"></param>
-      /// <param name="addRows"></param>
-      void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows);
-      /// <summary>
-      /// Changes more than one row at a time. This is specially helpful, since changes make a huge
-      /// percentage of the pubsub traffic.
-      /// </summary>
-      /// <param name="rows"></param>
-      void batchChange (IBlueRowCollection rows);
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Xml.Serialization;
+
+using BCG.Utilities;
+
+using bts;
+
+namespace BCG {
+   /// <summary>
+   /// This class will be used by the clients to publish tabular data to the pubsub
+   /// </summary>
+   public class PubTableBroker {
+      protected PublisherCore _publisher_core;
+      protected IDictionary<string, SinglePubTableBroker> _publisher_map =
+         new Dictionary<string, SinglePubTableBroker>();
+
+      protected bool myWaitingForFirstOp = false;
+
+      public PubTableBroker()
+         : this(new PublisherCore()) { }
+
+      public PubTableBroker(PublisherCore publisher) {
+         _publisher_core = publisher;
+         Log.Gen.info("PubTableBroker constructed");
+      }
+
+      public virtual void create_table(IBlueRowCollection table) {
+         string log_msg = $"PubTableBroker.createTable table: {table.schema.tableName}\n";
+         foreach (IBlueRow r in table) {
+            log_msg += $"id: {r["id"]}";
+         }
+         bts.Log.Gen.info(log_msg);
+         myCheckTransaction();
+         SinglePubTableBroker table_publisher = null;
+         var name_key = table.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
+            table_publisher = new SinglePubTableBroker(name_key, _publisher_core);
+            _publisher_map.Add(name_key, table_publisher);
+         }
+         table_publisher.create_table(table);
+      }
+
+      /// <summary>
+      /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
+      /// will be thrown
+      /// </summary>
+      /// <param name="r"></param>
+      public virtual void make_batch_changes(IBlueRowCollection new_rows,
+                                             IBlueRowCollection del_rows,
+                                             IBlueRowCollection added_rows) {
+         string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}";
+         foreach (IBlueRow r in del_rows)
+            log_msg += $"id: {r["id"]} ";
+         log_msg += "addedRows: ";
+         foreach (IBlueRow r in added_rows)
+            log_msg += $"id: {r["id"]}";
+         bts.Log.Gen.info(log_msg);
+         SinglePubTableBroker table_publisher = null;
+         var name_key = new_rows.schema.tableName.ToLower();
+         if (!_publisher_map.TryGetValue(name_key, out table_publisher))
+            throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
+         else
+            table_publisher.make_batch_changes(del_rows, added_rows);
+      }
+
+      protected void myCheckTransaction() {
+         if (myWaitingForFirstOp)
+            _publisher_core.begin_transaction();
+         myWaitingForFirstOp = false;
+      }
+   }
+}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs

@@ -1,162 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-namespace BCG {
-   /// <summary>
-   /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer 
-   /// that it initializes according to the table, to optimize serialiazation, hence bw and performance
-   /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
-   /// ie the first call will always be create.
-   /// </summary>
-   public class PubTablePublisher : IPubTablePublisher {
-      private readonly string _tag;
-      protected IBlueDataSerializer mySerializer;
-      protected PublisherCore myPubCore;
-
-      public PubTablePublisher(string tag, PublisherCore publisherCore) {
-         _tag = tag;
-         bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}");
-         myPubCore = publisherCore;
-         mySerializer = new BlueDataSerializationController();
-      }
-
-      #region IPubTablePublisher Members
-
-      public virtual void addRow(IBlueRow row) {
-         byte[] data;
-         mySerializer.serialize(StreamOpCode.add,
-                                 myGetRowCollectionArray(row),
-                                 out data);
-         myPublish(data, row.schema.tableName);
-      }
-
-      public virtual void deleteRow(IBlueRow row) {
-         byte[] data;
-         mySerializer.serialize(StreamOpCode.remove,
-                                 myGetRowCollectionArray(row),
-                                 out data);
-         myPublish(data, row.schema.tableName);
-      }
-
-      public virtual void changeRow(IBlueRow row) {
-         byte[] data;
-         mySerializer.serialize(StreamOpCode.change,
-                                 myGetRowCollectionArray(row),
-                                 out data);
-         myPublish(data, row.schema.tableName);
-      }
-
-      public virtual void createTable(IBlueRowCollection rows) {
-         byte[] data;
-         mySerializer.serialize(StreamOpCode.create,
-                                 new IBlueRowCollection[] { rows },
-                                 out data);
-         bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
-         myPublish(data, rows.schema.tableName);
-         // byte[] ops;
-         // Array rows_ds;
-         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
-      }
-
-      //private ClientDeserializationController _test_deserializer = new ClientDeserializationController();
-      public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
-         byte[] data = null;
-         if (myIsRealChange(delRows, addRows))
-            batchChange(addRows);
-         else {
-            mySerializer.serialize(StreamOpCode.multipleRemoveAdd,
-                                    new IBlueRowCollection[] { delRows, addRows },
-                                    out data);
-            myPublish(data, delRows.schema.tableName);
-         }
-         // byte[] ops;
-         // Array rows_ds;
-         // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
-      }
-
-      public virtual void batchChange(IBlueRowCollection rows) {
-         byte[] data;
-
-         mySerializer.serialize(StreamOpCode.multipleChange,
-                                 new IBlueRowCollection[] { rows },
-                                 out data);
-         myPublish(data, rows.schema.tableName);
-      }
-
-      #endregion
-
-      #region private methods
-      private void myPublish(byte[] data, string tableName) {
-         bts.Log.Gen.info($"PubTablePublisher.myPublish table: {tableName} data: {data.Length}");
-         if (data.Length > 0)
-            myPubCore.publish("DataStreamTag",
-                                 PubStreamPrefixer.prefix(tableName),
-                                 new MemoryStream(data, 0, data.Length, false, true));
-      }
-
-      private IBlueRowCollection[] myGetRowCollectionArray(IBlueRow r) {
-         return new IBlueRowCollection[]{
-                                            BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r})
-                                         };
-      }
-
-      private bool myIsRealChange(IBlueRowCollection delRows, IBlueRowCollection addRows) {
-         if (delRows.Count == addRows.Count) {
-            foreach (IBlueRow dRow in delRows)
-               if (!addRows.containsID((int)dRow["id"]))
-                  return false;
-            return true;
-         } else
-            return false;
-      }
-      #endregion
-   }
-
-   public class IndependentTablePublisher : PubTablePublisher {
-      bool myIsCreated = false;
-
-      public IndependentTablePublisher(string tag, PublisherCore pubCore) : base(tag, pubCore) { }
-
-      public override void addRow(IBlueRow row) {
-         myCheckCreate(row.schema);
-         base.addRow(row);
-      }
-
-      public override void batchChange(IBlueRowCollection rows) {
-         myCheckCreate(rows.schema);
-         base.batchChange(rows);
-      }
-
-      public override void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
-         myCheckCreate(delRows.schema);
-         base.batchRemoveAdd(delRows, addRows);
-      }
-
-      public override void changeRow(IBlueRow row) {
-         myCheckCreate(row.schema);
-         base.changeRow(row);
-      }
-
-      public override void deleteRow(IBlueRow row) {
-         myCheckCreate(row.schema);
-         base.deleteRow(row);
-      }
-
-      public override void createTable(IBlueRowCollection rows) {
-         myIsCreated = true;
-         base.createTable(rows);
-      }
-
-
-      private void myCheckCreate(IBlueTableSchema schema) {
-         if (!myIsCreated) {
-            //so that the initiazliation work is done on the serializer
-            byte[] placeHolder;
-            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { new BlueTable(schema) }, out placeHolder);
-            myIsCreated = true;
-         }
-      }
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs

@@ -1,57 +0,0 @@
-using System;
-
-namespace BCG {
-   /// <summary>
-   /// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
-   /// the table name, from the config, and wraps around it. The external code that will deal with user calls for
-   /// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
-   /// serialization.
-   /// </summary>
-   public class BlueDataSerializationController : IBlueDataSerializer {
-      private IBlueDataSerializer mySerializer;
-
-      public BlueDataSerializationController() { }
-
-      #region IBlueDataSerializer Members
-
-      /// <summary>
-      /// Serializes an op and data; however if a non create is called before a create, 
-      /// the serializedData is set to an empty byte array. 
-      /// </summary>
-      /// <param name="op"></param>
-      /// <param name="rows"></param>
-      /// <param name="serializedData"></param>
-      public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) {
-         if (op == StreamOpCode.create) {
-            if (mySerializer == null)
-               mySerializer = myGetSerializerForTable(rows[0].schema);
-
-            mySerializer.serialize(op, rows, out serializedData);
-         } else {
-            if (mySerializer != null) {
-               mySerializer.serialize(op, rows, out serializedData);
-            } else {
-               serializedData = new byte[] { };
-            }
-         }
-      }
-      #endregion
-
-      private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) {
-         // TODOds
-         return new UniversalSerializer(theSchema);
-         string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName);
-
-         object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName);
-
-         if (serializerArgs != null)
-            return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer;
-         else
-            return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer;
-
-
-
-      }
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs

@@ -1,111 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-namespace BCG
-{
-    /// <summary>
-    /// Abstract base class that can potentially serve all implementations of IBlueDataSerializer. The IBlueRowSerializer
-    /// is used to serialize individual rows. The implementaions should, in the constructor, contruct the myRowSerializer
-    /// to do the specific row serializations. Every nonabtract implementation of the this, should have a contructor
-    /// taking in an IBlueTableSchema.
-    /// </summary>
-    public abstract class BlueDataSerializer : IBlueDataSerializer
-    {
-        protected IBlueRowSerializer myRowSerializer;
-        protected string myClientDeserializerName, myPubsubDeserializerName;
-
-        public BlueDataSerializer(string pubsubDeserializerName,
-                                   string clientDeserializerName)
-        {
-            myPubsubDeserializerName = pubsubDeserializerName;
-            myClientDeserializerName = clientDeserializerName;
-        }
-
-
-        #region IBlueDataSerializer Members
-        /// <summary>
-        /// Provides the following serialization
-        /// 
-        /// -add, remove, change:
-        ///   op
-        ///   serializedRow (depending on the implementation of myRowSerializer)
-        ///
-        /// -create
-        ///   op
-        ///   pubsubDeserializer name
-        ///   clientDeserializer name
-        ///   schema
-        ///   # of rows
-        ///   serializedRows
-        /// 
-        /// -multipleAddRemoves
-        ///   op
-        ///   # of deleted rows
-        ///   serializedRows
-        ///   # of added rows
-        ///   serializedRows
-        /// 
-        /// -multipleChanges  
-        ///   op
-        ///   # of changed rows
-        ///   serializedRows
-        /// </summary>
-        /// <param name="op"></param>
-        /// <param name="rows"></param>
-        /// <param name="serializedData"></param>
-        public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData)
-        {
-            // BlueTrace.WriteSerializeStart(op, rows.Length); 
-
-            BinaryWriter bw = new BinaryWriter(new MemoryStream());
-            if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
-            {
-                bw.Write(op);
-                myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw);
-            }
-            else if (op == StreamOpCode.create)
-            {
-                IBlueRowCollection createdRows = rows[0];
-                bw.Write(op);
-                bw.Write(myPubsubDeserializerName);
-                bw.Write(myClientDeserializerName);
-                int dummyIdCol = 0;
-                int[] dummyIdList = new int[1];
-                //the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order
-                IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol);
-                sortedSchema.serialize(bw);
-                bw.Write(createdRows.Count);
-                foreach (IBlueRow r in createdRows)
-                    myRowSerializer.serializeRow(r, StreamOpCode.add, bw);
-
-            }
-            else if (op == StreamOpCode.multipleRemoveAdd)
-            {
-                bw.Write(op);
-                IBlueRowCollection delRows = rows[0],
-                                     addRows = rows[1];
-                bw.Write(delRows.Count);
-                foreach (IBlueRow dr in delRows)
-                    myRowSerializer.serializeRow(dr, StreamOpCode.remove, bw);
-                bw.Write(addRows.Count);
-                foreach (IBlueRow ar in addRows)
-                    myRowSerializer.serializeRow(ar, StreamOpCode.add, bw);
-            }
-            else if (op == StreamOpCode.multipleChange)
-            {
-                bw.Write(op);
-                IBlueRowCollection changedRows = rows[0];
-                bw.Write(changedRows.Count);
-                foreach (IBlueRow cr in changedRows)
-                    myRowSerializer.serializeRow(cr, StreamOpCode.change, bw);
-            }
-            bw.BaseStream.Position = 0;
-            serializedData = (bw.BaseStream as MemoryStream).ToArray();
-
-            // BlueTrace.WriteSerializeStop(serializedData.Length);
-        }
-        #endregion
-    }
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs

@@ -1,21 +0,0 @@
-using System;
-using System.Collections;
-
-namespace BCG
-{
-	/// <summary>
-	/// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[].
-	/// </summary>
-	public interface IBlueDataSerializer
-	{
-      /// <summary>
-      /// f(op, rows) = byte[]
-      /// </summary>
-      /// <param name="op">Operation that can be add, delete, change etc</param>
-      /// <param name="rows">The data to serialize. The only case when it will be an array of more than 1 element
-      ///                    is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd.
-      /// </param>
-      /// <param name="serializedData">The data serialized as bytes to be send over the wire </param>
-      void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs

@@ -1,23 +0,0 @@
-using System;
-using System.IO;
-
-namespace BCG
-{
-	/// <summary>
-	/// Responsible for serilializing rows
-    /// The way the serialization scheme works is that the header is same
-	/// for all serialization algorithms, only the way the row is serialized is different. Abstracting it out and we
-	/// can have each specific serializer have a unique implementation of IBlueRowSerializer. 
-	/// </summary>
-	public interface IBlueRowSerializer
-	{
-      /// <summary>
-      /// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate
-      /// for one row; the ones allowed are StreamopCode.add, remove and change only.
-      /// </summary>
-      /// <param name="r"></param>
-      /// <param name="bw"></param>
-      void serializeRow (IBlueRow r, byte op, BinaryWriter bw);
-
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs

@@ -1,96 +0,0 @@
-using System;
-using System.IO;
-
-namespace BCG
-{
-	/// <summary>
-	/// Serializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1)
-	/// rather than searching it up and making it O(n), n = types supported.
-	/// </summary>
-	public class PrimitiveSerializationHelper
-	{
-      private delegate void SerializeDelegate (object data, BinaryWriter bw);
-      private SerializeDelegate[] myDispatchTable;
-		
-      #region constructors
-      public PrimitiveSerializationHelper (IBlueTableSchema schema)
-         : this (schema.columnInfo) {}
-
-      public PrimitiveSerializationHelper (ColumnInfo[] cis)
-      {
-         myDispatchTable = new SerializeDelegate [cis.Length];
-         foreach (ColumnInfo ci in cis)
-            myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType);
-      }
-      #endregion
-
-      #region public methods
-      /// <summary>
-      /// Serializes the column no in the row and writes it into the BinaryWriter
-      /// </summary>
-      /// <param name="r"></param>
-      /// <param name="colNo"></param>
-      /// <param name="bw"></param>
-      public void serializeColumn (IBlueRow r, int colNo, BinaryWriter bw)
-      {
-         myDispatchTable[colNo] (r[colNo], bw);
-      }
-
-      /// <summary>
-      /// Writes the data to the BinaryWriter. The ColNo is provided to get the appropriate
-      /// serializer.
-      /// </summary>
-      /// <param name="data"></param>
-      /// <param name="colNo"></param>
-      /// <param name="bw"></param>
-      public void serializeColumn (object data, int colNo, BinaryWriter bw)
-      {
-         myDispatchTable[colNo] (data, bw);
-      }
-      #endregion
-
-      #region private methods
-      private void mySerializeInt32 (object data, BinaryWriter bw) {bw.Write ((Int32)data);}
-      private void mySerializeDouble (object data, BinaryWriter bw) {bw.Write ((double)data);}
-      private void mySerializeString (object data, BinaryWriter bw){bw.Write (data as string);}
-      private void mySerializeByteArray (object data, BinaryWriter bw)
-      {
-         byte[] bs = data as byte[];
-         bw.Write (bs.Length);
-         bw.Write (bs);
-      }
-      private void mySerializeChar (object data, BinaryWriter bw) {bw.Write ((char)data);}
-      private void mySerializeLong (object data, BinaryWriter bw) {bw.Write ((long)data);}
-      private void mySerializeSByte (object data, BinaryWriter bw) {bw.Write ((sbyte)data);}
-      private void mySerializeBool (object data, BinaryWriter bw) {bw.Write ((bool)data);}
-      private void mySerializeByte (object data, BinaryWriter bw) {bw.Write ((byte)data);}
-      private void mySerializeDecimal (object data, BinaryWriter bw) {bw.Write ((decimal)data);}
-      private void mySerializeUInt (object data, BinaryWriter bw) {bw.Write ((uint)data);}
-      private void mySerializeULong (object data, BinaryWriter bw){bw.Write ((ulong)data);}
-      private void mySerializeDateTime (object data, BinaryWriter bw) {bw.Write (((DateTime)data).Ticks);}
-      private void mySerializeSingle (object data, BinaryWriter bw) {bw.Write ((Single)data);}
-      private void mySerializeISerializedType (object data, BinaryWriter bw) {(data as ISerializedType).serialize (bw);}
-
-      private SerializeDelegate myGetDispatchMethod (Type t)
-      {
-         if (t.Equals (BlueDataValidTypes.INT)) return new SerializeDelegate (mySerializeInt32);
-         else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new SerializeDelegate (mySerializeDouble);
-         else if (t.Equals (BlueDataValidTypes.STRING)) return new SerializeDelegate (mySerializeString);
-         else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new SerializeDelegate (mySerializeByteArray);
-         else if (t.Equals (BlueDataValidTypes.CHAR)) return new SerializeDelegate (mySerializeChar);
-         else if (t.Equals (BlueDataValidTypes.LONG)) return new SerializeDelegate (mySerializeLong);
-         else if (t.Equals (BlueDataValidTypes.SBYTE)) return new SerializeDelegate (mySerializeSByte);
-         else if (t.Equals (BlueDataValidTypes.BOOL)) return new SerializeDelegate (mySerializeBool);
-         else if (t.Equals (BlueDataValidTypes.BYTE)) return new SerializeDelegate (mySerializeByte);
-         else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new SerializeDelegate (mySerializeDecimal);
-         else if (t.Equals (BlueDataValidTypes.UINT)) return new SerializeDelegate (mySerializeUInt);
-         else if (t.Equals (BlueDataValidTypes.ULONG)) return new SerializeDelegate (mySerializeULong);
-         else if (t.Equals (BlueDataValidTypes.DATETIME)) return new SerializeDelegate (mySerializeDateTime);
-         else if (t.Equals (BlueDataValidTypes.SINGLE)) return new SerializeDelegate (mySerializeSingle);
-         else if (SerializationUtilityMethods.isSerializedType (t)) return new SerializeDelegate (mySerializeISerializedType);
-         else  throw new ApplicationException (t.FullName + " is not a supported type");
-      }
-      #endregion
-   
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs

@@ -1,120 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-using BCG.Utilities;
-using System.Collections.Generic;
-namespace BCG
-{
-    /// <summary>
-    /// Helps the UniversalSerializer by serializing one row. Hence this class is agnostic to the operation
-    /// which is figured out by UniversalSerializer. 
-    /// 
-    /// it sorts the schema it gets, to make sure that the rows are always published the same way. 
-    /// For now, the sorting operation doesn't do anything. The reason for this is that C++ clients 
-    /// are depending upon the data arriving in a certain order. 
-    /// 
-    /// It performs a mapping : (row)->byte[]
-    /// </summary
-    internal class UniversalRowSerializer : IBlueRowSerializer
-    {
-        private PrimitiveSerializationHelper myPrimitiveSerializationHelper;
-        private int myFixedSize;
-        private int[] myNonFixedColIndexes;
-        private IBlueTableSchema mySchema;
-        //this null map is the null map that will be written to the wire
-        //therefore, its values represent the nulls in row, ordered by the-rearranged schema. 
-        private BitArray myNullMap;
-        private byte[] myNullMapArray;   //just so that a new instance is not created everytime
-        //this thing stores a mapping between the columns in the original schema that were passed in
-        private int[] mySortedSchemaToOriginalSchemaMapping;
-        private int myIdColumnInSortedSchema;  //this is used because the id column will be different in the original row and the schema
-
-        public UniversalRowSerializer(IBlueTableSchema schema)
-        {
-            mySchema = schema;
-            myPrimitiveSerializationHelper = new PrimitiveSerializationHelper(mySchema);                                
-           //therefore well use the original code
-           //sort the schema after we give it to the serialiaztion helper, because the helper constructs a dispatch table
-            //so it needs to know about the original schema. 
-            mySchema = BlueDataUtility.getSortedSchema(mySchema, out mySortedSchemaToOriginalSchemaMapping, out myIdColumnInSortedSchema);
-            myNullMapArray =
-               new byte[Convert.ToInt32(Math.Ceiling(Convert.ToDouble(mySchema.columnInfo.Length) / 8))];
-            myNullMap =
-               new BitArray(myNullMapArray);
-            myInitSizeCalculations();
-        }
-
-        #region IBlueRowSerializer methods
-        /// <summary>
-        /// Row serialized in the following order
-        /// row size (nullBitMap size + actual row size)
-        /// null bit map
-        /// non null row members (with id as the first col)
-        /// </summary>
-        /// <param name="r"></param>
-        /// <param name="bw"></param>
-        public void serializeRow(IBlueRow r, byte op, BinaryWriter bw)
-        {
-           myNullMap.SetAll(false);
-           for (int i = 0; i < mySchema.columnNames.Length; i++)
-              if (r[mySortedSchemaToOriginalSchemaMapping[i]] == DBNull.Value || r[mySortedSchemaToOriginalSchemaMapping[i]] == null)
-                 myNullMap[i] = true;
-           myNullMap.CopyTo(myNullMapArray, 0);
-           long initPos = bw.BaseStream.Position;
-           bw.Write((int)0);  ///place holder for size           
-           bw.Write(myNullMapArray);
-           bw.Write((int)r[mySortedSchemaToOriginalSchemaMapping[myIdColumnInSortedSchema]]);
-           foreach (ColumnInfo ci in mySchema.columnInfo)
-              if (!myNullMap[ci.colNo] && ci.colNo != myIdColumnInSortedSchema)
-                 myPrimitiveSerializationHelper.serializeColumn(r, mySortedSchemaToOriginalSchemaMapping[ci.colNo], bw);
-           long endPos = bw.BaseStream.Position;
-           bw.BaseStream.Position = initPos;
-           bw.Write(Convert.ToInt32(endPos - (initPos+4))); /// size does not include the meta data hence substracting 4
-           bw.BaseStream.Position = endPos;
-        }
-
-        public void serializeSchema(BinaryWriter bw)
-        {
-            mySchema.serialize(bw);
-        }
-        #endregion
-     
-        #region size calculations
-        /// <summary>
-        /// Sets up the fixed length (size in bytes occupied by each of the primitive type)
-        /// and an index for the non primitive type
-        /// </summary>
-        private void myInitSizeCalculations()
-        {
-            ArrayList temp = new ArrayList();
-            myFixedSize = myNullMapArray.Length;
-            foreach (ColumnInfo ci in mySchema.columnInfo)
-            {
-                if (!SerializationUtilityMethods.isFixedLengthType(ci.colType))
-                    temp.Add(ci.colNo);
-                else
-                    myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType);
-            }
-            myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[];
-        }
-
-        /// <summary>
-        /// Find the size of the serialized row, assumes that nullMap has already
-        /// been built
-        /// </summary>
-        /// <param name="fixedSize"></param>
-        /// <param name="r"></param>
-        /// <returns></returns>
-        private int myGetRowSizeInBytes(int fixedSize, IBlueRow r)
-        {
-            foreach (int colNo in myNonFixedColIndexes)
-            {
-                if (!myNullMap[colNo])
-                    fixedSize += //myGetStringSizeInBytes (r[colNo] as string);
-                       SerializationUtilityMethods.getObjectSerializedSize(r[mySortedSchemaToOriginalSchemaMapping[colNo]]);
-            }
-            return fixedSize;
-        }
-        #endregion
-    }
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs

@@ -1,16 +0,0 @@
-using System;
-
-namespace BCG
-{
-   /// <summary>
-   /// Uses the UniversalRowSerializer as the IBlueRowSerializer implementation.
-   /// </summary>
-	public class UniversalSerializer : BlueDataSerializer
-	{
-		public UniversalSerializer (IBlueTableSchema schema)
-          : base ("BCG.PubsubUniversalDeserializer, DataStreamCore", "BCG.ClientUniversalDeserializer")
-		{
-         myRowSerializer = new UniversalRowSerializer (schema);
-		}
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/Publish/SinglePubTableBroker.cs

@@ -0,0 +1,54 @@
+using bts;
+
+using System;
+using System.Collections;
+using System.IO;
+
+namespace BCG {
+   /// <summary>
+   /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer 
+   /// that it initializes according to the table, to optimize serialiazation, hence bw and performance
+   /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
+   /// ie the first call will always be create.
+   /// </summary>
+   public class SinglePubTableBroker {
+      private readonly string _tag;
+      protected IStreamIOMsgSerializer _serializer;
+      protected PublisherCore _publisher_core;
+
+      public SinglePubTableBroker(string tag, PublisherCore publisher_core) {
+         bts.Log.Gen.info($"SinglePubTableBroker.cons created for tag: {tag}");
+         _tag = tag;
+         _publisher_core = publisher_core;
+         _serializer = new UniversalStreamIOSerializer();
+      }
+
+      public void create_table(IBlueRowCollection rows) {
+         bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(rows);
+         bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
+         byte[] msg_bytes = _serializer.serialize(stream_io_msg);
+         PublishMsg pub_msg = new PublishMsg() {
+            tag = DSTagManager.get_stream_tag(rows.schema.tableName),
+            tag_type = DSTagManager.DataStreamTagType,
+            data_to_publish = msg_bytes
+         };
+         PubSubClientMessage msg = new PubSubClientMessage(pub_msg);
+         _publisher_core.publish(msg);
+      }
+
+      public void make_batch_changes(IBlueRowCollection del_rows, IBlueRowCollection added_rows) {
+         bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
+            new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
+         bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(mra_msg);
+         byte[] msg_bytes = _serializer.serialize(stream_io_msg);
+         PublishMsg pub_msg = new PublishMsg() {
+            tag = DSTagManager.get_stream_tag(added_rows.schema.tableName),
+            tag_type = DSTagManager.DataStreamTagType,
+            data_to_publish = msg_bytes
+         };
+         PubSubClientMessage msg = new PubSubClientMessage(pub_msg);
+         _publisher_core.publish(msg);
+      }
+
+   }
+}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs

@@ -1,25 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-
-namespace BCG
-{
-	/// <summary>
-	/// Deserializes a row. Returns the id of the row and its full content
-	/// </summary>
-	public interface IPubsubRowDeserializer
-	{
-      /// <summary>
-      /// Reads the data from the BinaryReader and deserializes it in the form of a row
-      /// </summary>
-      /// <param name="currentState">Some deserializers might need the current state, ie a Dictionary of rows
-      /// keyed off the id</param>
-      /// <param name="br">BinaryReader starting at the row</param>
-      /// <param name="op">The operation, as this might drive the deserializations scheme in some cases. This op
-      /// however is one of the basics operations, add remove or change as all other operations can be resolved into this</param>
-      /// <param name="id">The id of the deserialized row</param>
-      /// <param name="row">The actual row contents</param>
-      void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row);
-	}
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs

@@ -1,168 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-using BCG.Utilities;
-using System.Text;
-
-namespace BCG
-{
-   /// <summary>
-   /// Base class for all pubsubs deserializers. 
-   /// </summary>
-	public abstract class PubsubDeserializer : IPubsubDeserializer
-	{
-      // this is where the stream state is stored in pubsub?
-      protected IDictionary myState = new Hashtable ();
-      protected IBlueTableSchema mySchema;
-      protected string myClientDeserializerName;   //for the getState method
-      protected IPubsubRowDeserializer myDeserializer;
-		
-      public PubsubDeserializer () {}
-      
-      #region IPubSubDeserializer Members
-      public virtual void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams));
-         ArrayList ops = new ArrayList (), rows = new ArrayList ();
-         while (true)
-         {
-            IDictionary[] result;
-            byte op;
-            try 
-            {
-               int value = br.BaseStream.ReadByte();
-               if (value == -1) break;
-               op = (byte)value;
-            }
-            catch (EndOfStreamException) 
-            {
-               // TODO, logically break loop
-               break;
-            }
-            if (op == StreamOpCode.create)
-            {
-               myState.Clear ();
-               br.ReadString (); //the pubsub-deserializer name, should be the same as this's name 
-               myClientDeserializerName = br.ReadString (); //the client-deserializer name
-               (mySchema = new BlueTableSchema()).deserialze (br);
-               result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())};
-               myEncorporatRows (result[0], op);
-               ops.Add (op);
-               rows.Add (result);
-            }
-            else
-            {
-               if (mySchema != null)
-               {
-                  if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
-                  {
-                     result = new IDictionary[] {myGetRows (br, op, 1)};
-                     myEncorporatRows (result[0], op);
-                     ops.Add (op);
-                     rows.Add (result);
-                  }
-                  else if (op == StreamOpCode.multipleChange)
-                  {
-                     result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())};
-                     myEncorporatRows (result[0], op);
-                     ops.Add (op);
-                     rows.Add (result);
-                  }
-                  else if (op == StreamOpCode.multipleRemoveAdd)
-                  {
-                     ops.Add (op);
-                     result = new IDictionary [] {
-                                                    myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()),
-                                                    myGetRows (br, StreamOpCode.add, br.ReadInt32 ())
-                                                 };
-                     myEncorporatRows (result[0], StreamOpCode.remove);
-                     myEncorporatRows (result[1], StreamOpCode.add);
-                     rows.Add (result);
-                  }
-                  else
-                  {
-                     // BlueEventLog.logEvent ("Illegal op-code received", BlueEventLevel.Error);
-                  }
-               }
-               else
-               {
-                  // BlueEventLog.logEvent ("Received a non create operation even though the schema" +
-                  //    "has never been created", BlueEventLevel.Warning);
-               }
-            }
-         }
-         theOps = ops.ToArray (typeof(byte)) as byte[];
-         theRows = rows.ToArray ();
-      }
-
-      /// <summary>
-      /// Sends in a create signal, so that new clients can get a create everytime and start
-      /// afresh from there
-      /// </summary>
-      /// <returns></returns>
-      public virtual byte[] getState()
-      {
-         if (mySchema == null)
-            return new byte[] {};
-         else
-         {
-            BinaryWriter bw = new BinaryWriter (new MemoryStream ());
-            bw.Write (StreamOpCode.create);
-            bw.Write (GetType().AssemblyQualifiedName);
-            bw.Write (myClientDeserializerName);
-            mySchema.serialize (bw);
-            bw.Write (myState.Count);
-            foreach (byte[] row in myState.Values)
-               bw.Write (row);
-            return (bw.BaseStream as MemoryStream).ToArray ();
-         }
-      }
-      #endregion
-
-      /// <summary>
-      /// Every row that has been deserialized passes through this method, so that it can update the state.
-      /// </summary>
-      /// <param name="rows"></param>
-      /// <param name="op"></param>
-      protected virtual void myEncorporatRows (IDictionary rows, byte op)
-      {
-         if (op == StreamOpCode.add || op == StreamOpCode.create)
-            foreach (DictionaryEntry de in rows)
-            {
-               if (myState.Contains(de.Key)) {
-                  // BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key),BlueEventLog2Level.Fatal);
-                  StringBuilder b = new StringBuilder();
-                  foreach (string n in mySchema.columnNames)
-                     b.AppendFormat(" {0}", n);
-                  // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
-               }
-               myState.Add(de.Key, de.Value);
-            }
-         else if (op == StreamOpCode.remove)
-            foreach (DictionaryEntry de in rows)
-            {
-               if (!myState.Contains(de.Key))
-               {
-                  // BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
-                  StringBuilder b = new StringBuilder();
-                  foreach (string n in mySchema.columnNames)
-                     b.AppendFormat(" {0}", n);
-                  // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()),BlueEventLog2Level.Fatal);
-               }
-               myState.Remove(de.Key);
-            }
-         else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
-            foreach (DictionaryEntry de in rows)
-               myState[de.Key] = de.Value;
-      }
-
-      /// <summary>
-      /// Mass deserialization happens here, also since there is no knowledge of the schema 
-      /// </summary>
-      /// <param name="br"></param>
-      /// <param name="op"></param>
-      /// <param name="noOfRows"></param>
-      /// <returns></returns>
-      protected abstract IDictionary myGetRows (BinaryReader br, byte op, int noOfRows);
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs

@@ -1,23 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-namespace BCG {
-   public class PubsubUniversalDeserializer : PubsubDeserializer {
-      public PubsubUniversalDeserializer() {
-         bts.Log.Gen.info("PubsubUniversalDeserializer.cons");
-      }
-
-      protected override IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) {
-         myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
-         Hashtable retVal = new Hashtable();
-         for (int i = 0; i < noOfRows; i++) {
-            int id; object row;
-            myDeserializer.deserializeRow(myState, br, op, out id, out row);
-            retVal.Add(id, row);
-         }
-         return retVal;
-      }
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs

@@ -1,34 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-
-
-namespace BCG
-{
-   public class PubsubUniversalRowDeserializer : IPubsubRowDeserializer
-   {
-      private int myNullMapLength; 
-      private IBlueTableSchema mySchema;
-      private int myRowOverheadSize = SerializationUtilityMethods.getFixedLengthTypeSize (typeof(int));
-      
-      public PubsubUniversalRowDeserializer (IBlueTableSchema schema)
-      {
-         mySchema = schema;
-         myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
-      }
-
-      #region IRowDeserializer Members
-
-      public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row)
-      {
-         //current state is ignored
-         long pos = br.BaseStream.Position;
-         int rowSize = br.ReadInt32 ();
-         br.BaseStream.Position = pos;
-         row = br.ReadBytes (rowSize + myRowOverheadSize);
-         id = BitConverter.ToInt32 (row as byte[], myNullMapLength + myRowOverheadSize);
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs

@@ -1,49 +0,0 @@
-using System;
-using System.IO;
-using System.Collections.Generic;
-using BCG.Utilities;
-
-namespace BCG {
-   /// <summary>
-   /// Keeps a collection of TagStateBanks to hold the state of each tag
-   /// </summary>
-	public class PubsubServerTagHandler : IPubSubServerTagHandler {
-      private IDictionary<string, TagStateBank> _tag_state_map = new Dictionary<string, TagStateBank>();
-      private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats;
-         false;
-
-      #region IPubSubServerTagHandler Members
-
-      public byte[] getContent(string tag) {
-         TagStateBank tag_state_bank;
-         if (_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
-            return tag_state_bank.getConent();
-         } else {
-            bts.Log.Gen.warn("Information requested for tag: {tag}, it does not exist now");
-            return new byte[] { };
-         }
-      }
-
-      public void update(string tag, Stream[] messageBodies) {
-         bts.Log.Gen.info($"PubsubServerTagHandler.update tag: {tag} streams: {messageBodies.Length}");
-         if (myPublishStatistics) {
-            StatisticsLogger.instance.log(tag, messageBodies);
-         }
-         TagStateBank tag_state_bank;
-         if (!_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
-            bts.Log.Gen.info("tag: {tag} received for the first time creating TagStateBank for it now");
-            tag_state_bank = new TagStateBank();
-            _tag_state_map.Add(tag, tag_state_bank);
-         }
-         tag_state_bank.update(messageBodies);
-      }
-
-      public void doWork() { }
-
-      public string[] supportedTagTypes { get { return new string[] { "DataStreamTag" }; } }
-
-      public void register(InternalPublishDelegate callback) { }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs

@@ -1,102 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-using BCG.Utilities;
-using System.Threading;
-
-namespace BCG
-{
-   class TagData
-   {
-      public DateTime lastUpdateTime;
-      public long totalSize;
-      public int lastUpdateSize;
-
-      public TagData(DateTime ut, long ts, int lus)
-      {
-         lastUpdateTime = ut;
-         totalSize = ts;
-         lastUpdateSize = lus;
-      }
-      public override string ToString()
-      {
-         return lastUpdateTime.ToString() + "," + totalSize.ToString() + "," + lastUpdateSize.ToString();
-      }
-   }
-
-   class StatisticsLogger
-   {
-      private IDictionary myTagData = new Hashtable();
-      private static StatisticsLogger mySelf;
-      private BlockingQueue myQueue = new BlockingQueue();
-      private DateTime myLastFireTime = DateTime.MinValue;
-      private TimeSpan myPubsubStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.pubsubStatsUpdateInterval);
-      private TimeSpan myTagStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.singleTagStatsUpdateInterval);
-
-      private StatisticsLogger()
-      {
-         Thread readThread = new Thread(new ThreadStart(myReadAndPublish));
-         readThread.Priority = ThreadPriority.BelowNormal;
-         readThread.Start();
-      }
-      public static StatisticsLogger instance
-      {
-         get
-         {
-            if (mySelf == null)
-               mySelf = new StatisticsLogger();
-            return mySelf;
-         }
-      }
-
-      public void log(string tag, Stream[] bodies)
-      {
-         long nBytes = 0;
-         foreach (Stream s in bodies)
-            nBytes += s.Length;
-         myQueue.enqueue(new object[] { tag, Convert.ToInt32 (nBytes) });
-      }
-
-      private void myReadAndPublish()
-      {
-         while (true)
-         {
-            object qe = myQueue.dequeue(50);
-            if (qe != null)
-            {
-               object[] args = qe as object[];
-               string tag = args[0].ToString();
-               int dataSize = (int)args[1];
-               if (!myTagData.Contains(tag))
-               {
-                  myTagData.Add(tag, new TagData(DateTime.Now, Convert.ToInt64(dataSize), dataSize));
-               }
-               else
-               {
-                  TagData tdToSend = myTagData[tag] as TagData;
-                  tdToSend.lastUpdateSize += dataSize;
-                  tdToSend.totalSize += dataSize;
-               }
-            }
-            myFireMonitors();
-         }
-      }
-
-      private void myFireMonitors()
-      {
-         if (DateTime.Now.Subtract(myLastFireTime) >= myPubsubStatsUpdateInterval)
-         {
-            foreach (string tag in myTagData.Keys)
-            {
-               TagData td = myTagData[tag] as TagData;
-               if (td.lastUpdateSize > 0 && DateTime.Now.Subtract(td.lastUpdateTime) >= myTagStatsUpdateInterval)
-               {
-                  td.lastUpdateTime = DateTime.Now;
-                  td.lastUpdateSize = 0;
-               }
-            }
-            myLastFireTime = DateTime.Now;
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs

@@ -1,26 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-
-namespace BCG {
-   public class TagStateBank {
-      private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController();
-
-      public TagStateBank() { }
-
-      public TagStateBank(string tag) {
-         bts.Log.Gen.info("TagStateBank created for tag: {tag}");
-      }
-
-      public byte[] getConent() {
-         return myDeserializationController.getState();
-      }
-
-      public void update(Stream[] streams) {
-         bts.Log.Gen.info($"TagStateBank.update streams: {streams.Length}");
-         byte[] ops;
-         Array rowCollection;
-         myDeserializationController.deserialize(streams, out ops, out rowCollection);
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs

@@ -1,146 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using BCG.Utilities;
-
-namespace BCG
-{
-   public class CompressionClientDeserializer : IClientDeserializer
-   {
-      protected IBlueTableSchema mySchema;
-      protected IBlueTableSchema myFirstSchema;
-      protected IClientRowDeserializer myRowDeserializer;
-
-      public CompressionClientDeserializer()
-      {
-         myFirstSchema = null;
-         mySchema = null;
-      }
-
-      #region IClientDeserializer Members
-
-      public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
-         // BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length);
-
-         // emit # of streams and total length
-         ArrayList ops = new ArrayList(), rows = new ArrayList();
-         IBlueRowCollection[] result;
-         while (true)
-         {
-            byte op;
-            try
-            {
-               int value = br.BaseStream.ReadByte();
-               if (value == -1) break;
-               op = (byte)value;
-               // BlueTrace.WriteDeserializeOperation(op);
-            }
-            catch (EndOfStreamException)
-            {
-               break;
-            }
-            if (op == StreamOpCode.create)
-            {
-               br.ReadString(); //pubsubDeserializer
-               br.ReadString(); //clientDeserializer
-               mySchema = new BlueTableSchema();
-               mySchema.deserialze(br);
-               //keep track of the first schema you've ever seen, so you know how to reorder columns
-               if (myFirstSchema == null)
-                  myFirstSchema = mySchema;
-
-               /// decompress here
-               int compressedMessageSize = br.ReadInt32();
-               byte[] compressedData = br.ReadBytes(compressedMessageSize);
-               BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
-
-               result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
-               myEncorporateRows(result[0], op);
-               ops.Add(op);
-               rows.Add(result);
-            }
-            else
-            {
-               if (mySchema != null)
-               {
-                  if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
-                  {
-                     result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
-                     myEncorporateRows(result[0], op);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else if (op == StreamOpCode.multipleChange)
-                  {
-                     int compressedMessageSize = br.ReadInt32();
-                     byte[] compressedData = br.ReadBytes(compressedMessageSize);
-                     BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
-
-                     result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
-                     myEncorporateRows(result[0], op);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else if (op == StreamOpCode.multipleRemoveAdd)
-                  {
-                     int compressedMessageSize = br.ReadInt32();
-                     byte[] compressedData = br.ReadBytes(compressedMessageSize);
-                     BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
-
-                     result =
-                        new IBlueRowCollection[] {
-                                                     myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()),
-                                                     myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ())
-                                                  };
-
-                     myEncorporateRows(result[0], StreamOpCode.remove);
-                     myEncorporateRows(result[1], StreamOpCode.add);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else
-                  {
-                     // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
-                  }
-               }
-               else
-               {
-                  // BlueEventLog.logEvent("Received a non create operation even though the schema" +
-                  //    "has never been created", BlueEventLevel.Warning);
-               }
-            }
-         }
-         theOps = ops.ToArray(typeof(byte)) as byte[];
-         theRows = rows.ToArray();
-         // BlueTrace.WriteDeserializeStop(rows.Count);
-      }
-
-      #endregion
-
-      private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows)
-      {
-         BlueTable retVal = new BlueTable(mySchema);
-
-         if (myRowDeserializer == null)
-            myRowDeserializer = new ClientUniversalRowDeserializer(mySchema);
-         else if (op == StreamOpCode.create)
-            myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema);
-
-         for (int i = 0; i < nRows; i++)
-         {
-            IBlueRow row;
-            myRowDeserializer.deserializeRow(null, br, op, out row);
-            retVal.addRow(row);
-         }
-         return retVal;
-      }
-
-      private void myEncorporateRows(IBlueRowCollection rows, byte op) { }
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs

@@ -1,192 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Collections;
-using BCG.Utilities;
-
-namespace BCG
-{
-   public class CompressionPubsubDeserializer : IPubsubDeserializer
-   {
-      protected IDictionary myState = new Hashtable();
-      protected IBlueTableSchema mySchema;
-      protected string myClientDeserializerName;   //for the getState method
-      protected IPubsubRowDeserializer myRowDeserializer;
-
-      public CompressionPubsubDeserializer() { }
-
-      #region IPubSubDeserializer Members
-      public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
-      {
-         BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
-         ArrayList ops = new ArrayList(), rows = new ArrayList();
-         while (true)
-         {
-            IDictionary[] result;
-            byte op;
-            try
-            {
-               int value = br.BaseStream.ReadByte();
-               if (value == -1) break;
-               op = (byte)value;
-            }
-            catch (EndOfStreamException)
-            {
-               break;
-            }
-            if (op == StreamOpCode.create)
-            {
-               myState.Clear();
-               br.ReadString(); //the pubsub-deserializer name, should be the same as this's name 
-               myClientDeserializerName = br.ReadString(); //the client-deserializer name
-               (mySchema = new BlueTableSchema()).deserialze(br);
-               /// deserialize this part
-               int compressedMessageSize = br.ReadInt32();
-               BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
-               result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
-               myEncorporatRows(result[0], op);
-               ops.Add(op);
-               rows.Add(result);
-            }
-            else
-            {
-               if (mySchema != null)
-               {
-                  if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
-                  {
-                     result = new IDictionary[] { myGetRows(br, op, 1) };
-                     myEncorporatRows(result[0], op);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else if (op == StreamOpCode.multipleChange)
-                  {
-                     int compressedMessageSize = br.ReadInt32();
-                     BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
-                     result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
-                     myEncorporatRows(result[0], op);
-                     ops.Add(op);
-                     rows.Add(result);
-                  }
-                  else if (op == StreamOpCode.multipleRemoveAdd)
-                  {
-                     ops.Add(op);
-                     int compressedMessageSize = br.ReadInt32();
-                     BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
-                     result = new IDictionary[] {
-                                                    myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()),
-                                                    myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ())
-                                                 };
-                     myEncorporatRows(result[0], StreamOpCode.remove);
-                     myEncorporatRows(result[1], StreamOpCode.add);
-                     rows.Add(result);
-                  }
-                  else
-                  {
-                     // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
-                  }
-               }
-               else
-               {
-                  // BlueEventLog.logEvent("Received a non create operation even though the schema" +
-                  //    "has never been created", BlueEventLevel.Warning);
-               }
-            }
-         }
-         theOps = ops.ToArray(typeof(byte)) as byte[];
-         theRows = rows.ToArray();
-      }
-
-      /// <summary>
-      /// Sends in a create signal, so that new clients can get a create everytime and start
-      /// afresh from there
-      /// </summary>
-      /// <returns></returns>
-      public virtual byte[] getState()
-      {
-         if (mySchema == null)
-            return new byte[] { };
-         else
-         {
-            BinaryWriter bw = new BinaryWriter(new MemoryStream());
-            bw.Write(StreamOpCode.create);
-            bw.Write(GetType().AssemblyQualifiedName);
-            bw.Write(myClientDeserializerName);
-            mySchema.serialize(bw);
-            /// compress it here
-
-            BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
-            decompressedBw.Write(myState.Count);
-            foreach (byte[] row in myState.Values)
-               decompressedBw.Write(row);
-            decompressedBw.BaseStream.Position = 0;
-            byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
-            bw.Write(compressedData.Length);
-            bw.Write(compressedData);
-
-            return (bw.BaseStream as MemoryStream).ToArray();
-         }
-      }
-      #endregion
-
-      /// <summary>
-      /// Every row that has been deserialized passes through this method, so that it can update the state.
-      /// </summary>
-      /// <param name="rows"></param>
-      /// <param name="op"></param>
-      protected virtual void myEncorporatRows(IDictionary rows, byte op)
-      {
-         if (op == StreamOpCode.add || op == StreamOpCode.create)
-            foreach (DictionaryEntry de in rows)
-            {
-               if (myState.Contains(de.Key))
-               {
-                  // BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
-                  StringBuilder b = new StringBuilder();
-                  foreach (string n in mySchema.columnNames)
-                     b.AppendFormat(" {0}", n);
-                  // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
-               }
-               myState.Add(de.Key, de.Value);
-            }
-         else if (op == StreamOpCode.remove)
-            foreach (DictionaryEntry de in rows)
-            {
-               if (!myState.Contains(de.Key))
-               {
-                  // BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
-                  StringBuilder b = new StringBuilder();
-                  foreach (string n in mySchema.columnNames)
-                     b.AppendFormat(" {0}", n);
-                  // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
-               }
-               myState.Remove(de.Key);
-            }
-         else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
-            foreach (DictionaryEntry de in rows)
-               myState[de.Key] = de.Value;
-      }
-
-      /// <summary>
-      /// Mass deserialization happens here, also since there is no knowledge of the schema 
-      /// </summary>
-      /// <param name="br"></param>
-      /// <param name="op"></param>
-      /// <param name="noOfRows"></param>
-      /// <returns></returns>
-      private IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
-      {
-         myRowDeserializer = myRowDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myRowDeserializer;
-         Hashtable retVal = new Hashtable ();
-         for (int i = 0; i < noOfRows; i++)
-         {
-            int id; object row;
-            myRowDeserializer.deserializeRow (myState, br, op, out id, out row);
-            retVal.Add (id, row);
-         }
-         return retVal;
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs

@@ -1,90 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-
-namespace BCG
-{
-   public class CompressionSerializer : IBlueDataSerializer
-   {
-      private IBlueRowSerializer myRowSerializer;
-      private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamCore", myClientDeserializerName = "BCG.CompressionClientDeserializer";
-      
-      public CompressionSerializer (IBlueTableSchema schema)
-      {
-         myRowSerializer = new UniversalRowSerializer (schema);
-      }
-
-      public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData)
-      {
-         // BlueTrace.WriteSerializeStart(op, rows.Length);
-         BinaryWriter bw = new BinaryWriter(new MemoryStream());
-         if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
-         {
-            bw.Write(op);
-            /// no need to compress the only row
-            myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw);
-         }
-         else if (op == StreamOpCode.create)
-         {
-            IBlueRowCollection createdRows = rows[0];
-            bw.Write(op);
-            bw.Write(myPubsubDeserializerName);
-            bw.Write(myClientDeserializerName);
-            int dummyIdCol = 0;
-            int[] dummyIdList = new int[1];
-            //the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order
-            IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol);
-            sortedSchema.serialize(bw);
-            /// compress it here
-            BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
-            decompressedBw.Write(createdRows.Count);
-            foreach (IBlueRow r in createdRows)
-               myRowSerializer.serializeRow(r, StreamOpCode.add, decompressedBw);
-            decompressedBw.BaseStream.Position = 0;
-
-            byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
-            bw.Write(compressedData.Length);
-            bw.Write(compressedData);
-         }
-         else if (op == StreamOpCode.multipleRemoveAdd)
-         {
-            bw.Write(op);
-            IBlueRowCollection delRows = rows[0], addRows = rows[1];
-            /// compress it here
-            BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
-            decompressedBw.Write(delRows.Count);
-            foreach (IBlueRow dr in delRows)
-               myRowSerializer.serializeRow(dr, StreamOpCode.remove, decompressedBw);
-            decompressedBw.Write(addRows.Count);
-            foreach (IBlueRow ar in addRows)
-               myRowSerializer.serializeRow(ar, StreamOpCode.add, decompressedBw);
-            decompressedBw.BaseStream.Position = 0;
-
-            byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
-            bw.Write(compressedData.Length);
-            bw.Write(compressedData);
-         }
-         else if (op == StreamOpCode.multipleChange)
-         {
-            bw.Write(op);
-            IBlueRowCollection changedRows = rows[0];
-            /// compress it here
-            BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
-            decompressedBw.Write(changedRows.Count);
-            foreach (IBlueRow cr in changedRows)
-               myRowSerializer.serializeRow(cr, StreamOpCode.change, decompressedBw);
-            decompressedBw.BaseStream.Position = 0;
-
-            byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
-            bw.Write(compressedData.Length);
-            bw.Write(compressedData);
-         }
-         bw.BaseStream.Position = 0;
-         serializedData = (bw.BaseStream as MemoryStream).ToArray();
-         // BlueTrace.WriteSerializeStop(serializedData.Length);
-      }     
-   }
-   
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs

@@ -1,11 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BCG
-{
-   public class BBOOptionDeserializer : ClientFixedDeserializer
-   {
-      public BBOOptionDeserializer() { }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs

@@ -1,49 +0,0 @@
-using System;
-using BCG.Utilities;
-
-namespace BCG
-{
-   public abstract class ClientFixedDeserializer : ClientDeserializer
-   {
-		private BlueTable myCurrentState;
-      
-      public ClientFixedDeserializer () {}
-
-      protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
-         op = op == StreamOpCode.create ? StreamOpCode.add :
-            op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
-         if (myRowDeserializer == null)
-         {
-            myRowDeserializer = new ClientFixedRowDeserializer (mySchema);
-            myCurrentState = new BlueTable (mySchema);
-         }
-         for (int i = 0; i < nRows; i++)
-         {
-            IBlueRow r;
-            myRowDeserializer.deserializeRow (myCurrentState, br, op, out r);
-            retVal.addRow (r);
-         }
-         return retVal;
-      }
-
-      protected override void myEncorporateRows (IBlueRowCollection rows, byte op) 
-      {
-         if (op == StreamOpCode.create)
-            myCurrentState = new BlueTable (rows);
-         else if (op == StreamOpCode.add)
-            myCurrentState.addRows (rows);
-         else if (op == StreamOpCode.remove)
-            myCurrentState.deleteRows (rows);
-         else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
-            myCurrentState.changeRows (rows);
-         else
-         {
-            // BlueEventLog.logAssert (false, "Illegal operation for this context");
-         }
-      }
-
-
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs

@@ -1,38 +0,0 @@
-using System;
-using System.IO;
-using BCG.Utilities;
-
-namespace BCG
-{
-	public class ClientFixedRowDeserializer : IClientRowDeserializer
-	{
-      private PrimitiveDeserializationHelper myHelper;
-      private IBlueTableSchema mySchema;
-		public ClientFixedRowDeserializer (IBlueTableSchema schema)
-		{
-         myHelper = new PrimitiveDeserializationHelper (mySchema = schema);
-      }
-
-      #region IClientRowDeserializer Members
-
-      public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
-      {
-         if (op == StreamOpCode.add || op == StreamOpCode.change)
-         {
-            object[] retVal = new object[mySchema.columnInfo.Length];
-            foreach (ColumnInfo ci in mySchema.columnInfo)
-               retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br);
-            r = new BlueRow (mySchema, retVal);
-         }
-         else if (op == StreamOpCode.remove)
-            r = currentState[br.ReadInt32 ()];
-         else
-         {
-            // BlueEventLog.logAssert (false, op.ToString() + " not valid in this context");
-            r = null;
-         }
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs

@@ -1,11 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BCG
-{
-   public class MarketOptionTheoClientDeserializer : ClientFixedDeserializer
-   {
-      public MarketOptionTheoClientDeserializer() { }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs

@@ -1,13 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BCG
-{
-   public class BBOOptionSerializer : FixedSerializer
-   {
-      public BBOOptionSerializer(IBlueTableSchema schema)
-         : base(schema, "BCG.BBOOptionPubsubDeserializer", "BCG.BBOOptionDeserializer")
-      { }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs

@@ -1,35 +0,0 @@
-using System;
-using System.IO;
-
-namespace BCG
-{
-	/// <summary>
-	/// Optimized to serialize fixed length rows, ie schemas that have only primitive types and non-nulls
-	/// </summary>
-	public class FixedRowSerializer : IBlueRowSerializer
-	{
-      private PrimitiveSerializationHelper myHelper;
-      private int myIdColNo;
-      private IBlueTableSchema mySchema;
-		public FixedRowSerializer (IBlueTableSchema schema)
-		{
-         myIdColNo = schema["id"].colNo;
-         myHelper = new PrimitiveSerializationHelper (schema);
-         mySchema = schema;
-      }
-
-      #region IBlueRowSerializer Members
-
-      public void serializeRow (IBlueRow r, byte op, BinaryWriter bw)
-      {
-         if (op == StreamOpCode.remove)
-            myHelper.serializeColumn (r, myIdColNo, bw);
-         else
-            foreach (ColumnInfo ci in r.schema.columnInfo)
-               myHelper.serializeColumn (r, ci.colNo, bw);
-      }
-
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs

@@ -1,20 +0,0 @@
-using System;
-using System.IO;
-
-namespace BCG
-{
-   /// <summary>
-   /// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls) 
-   /// </summary>
-   public abstract class FixedSerializer : BlueDataSerializer
-   {
-      public FixedSerializer (IBlueTableSchema schema, 
-                              string pubsubDeserializer,
-                              string clientDeserializer)
-         : base (pubsubDeserializer, clientDeserializer)
-      {
-         myRowSerializer = new FixedRowSerializer (schema);
-      }
-   }
-   
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs

@@ -1,14 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BCG
-{
-   public class MarketOptionTheoSerializer : FixedSerializer
-   {
-      public MarketOptionTheoSerializer(IBlueTableSchema schema)
-         : base(schema, "BCG.MarketOptionTheoPubsubDeserializer", 
-                        "BCG.MarketOptionTheoClientDeserializer")
-      { }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs

@@ -1,11 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BCG
-{
-   public class BBOOptionPubsubDeserializer : PubsubFixedDeserializer
-   {
-      public BBOOptionPubsubDeserializer() { }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs

@@ -1,12 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BCG
-{
-   public class MarketOptionTheoPubsubDeserializer : PubsubFixedDeserializer
-   {
-      public MarketOptionTheoPubsubDeserializer()
-      { }
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs

@@ -1,32 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-namespace BCG
-{
-   public abstract class PubsubFixedDeserializer : PubsubDeserializer
-   {
-      protected BlueTable myCurrentState;
-
-      public PubsubFixedDeserializer () {}
-
-      protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
-      {
-         if (myDeserializer == null)
-            myDeserializer = new PubsubFixedRowDeserializer (mySchema);
-         IDictionary retVal = new Hashtable ();
-         op = op == StreamOpCode.create ? StreamOpCode.add :
-            op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
-         for (int i = 0; i < noOfRows; i++)
-         {
-            int id;
-            object row;
-            myDeserializer.deserializeRow (myState, br, op, out id, out row);
-            retVal.Add (id, row);
-         }
-         return retVal;
-      }
-
-   }
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs

@@ -1,37 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-
-namespace BCG
-{
-	public class PubsubFixedRowDeserializer : IPubsubRowDeserializer
-   {
-      private int myRowSize, myIdOffset;
-
-      public PubsubFixedRowDeserializer (IBlueTableSchema schema)
-      {
-         foreach (ColumnInfo ci in schema.columnInfo)
-         {
-            myRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType);
-            myIdOffset += ci.colNo < schema["id"].colNo ? SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType) : 0;
-         }
-      }
-      #region IPubsubRowDeserializer Members
-
-      public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row)
-      {
-         if (op == StreamOpCode.add || op == StreamOpCode.change)
-         {
-            row = br.ReadBytes (myRowSize);
-            id = BitConverter.ToInt32 (row as byte[], myIdOffset);
-         }
-         else //delete => only the id
-         {
-            id = br.ReadInt32 ();
-            row = currentState[id];
-         }
-      }
-
-      #endregion
-   }
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs

@@ -1,56 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-
-using BCG.Utilities;
-
-namespace BCG
-{
-	public abstract class ClientFixedChangingDeserializer : ClientDeserializer
-	{
-      private string [] myChangingCols;
-      protected BlueTable myCurrentState;
-		
-      public ClientFixedChangingDeserializer (string[] changingCols)
-		{
-         myChangingCols = changingCols;
-		}
-
-      protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows)
-      {
-         BlueTable retVal = new BlueTable (mySchema);
-         op = op == StreamOpCode.create ? StreamOpCode.add :
-            op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
-         if (myRowDeserializer == null)
-         {
-            myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, myChangingCols);
-            myCurrentState = new BlueTable (mySchema);
-         }
-         for (int i = 0; i < nRows; i++)
-         {
-            IBlueRow r;
-            myRowDeserializer.deserializeRow (myCurrentState, br, op, out r);
-            retVal.addRow (r);
-         }
-         return retVal;
-      }
-
-      protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
-      {
-         if (op == StreamOpCode.create)
-            myCurrentState = new BlueTable (rows);
-         else if (op == StreamOpCode.add)
-            myCurrentState.addRows (rows);
-         else if (op == StreamOpCode.remove)
-            myCurrentState.deleteRows (rows);
-         else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
-            myCurrentState.changeRows (rows);
-         else
-         {
-            // BlueEventLog.logAssert (false, "Illegal operation for this context");
-         }
-      }
-
-
-	}
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs

@@ -1,67 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-
-using BCG.Utilities;
-
-namespace BCG
-{
-   /// <summary>
-   /// Deserializes a fixed length row with no nulls on the client side.
-   /// </summary>
-   public class ClientFixedChangingRowDeserializer : IClientRowDeserializer
-   {
-      private int[] myChangingColIndices;
-      protected PrimitiveDeserializationHelper myHelper;
-      protected IBlueTableSchema mySchema;
-
-      public ClientFixedChangingRowDeserializer (IBlueTableSchema schema, string[] changingCols)
-      {
-         myHelper = new PrimitiveDeserializationHelper (mySchema = schema);
-         myChangingColIndices = new int [changingCols.Length];
-         for (int i = 0; i < changingCols.Length; i++)
-            myChangingColIndices[i] = schema[changingCols[i]].colNo;
-         Array.Sort (myChangingColIndices);
-      }
-
-      public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
-      {
-         if (op == StreamOpCode.add)
-            myDeserializeAdd (currentState, br, out r);
-         else if (op == StreamOpCode.remove)
-            myDeserializeRemove (currentState, br, out r);
-         else if (op == StreamOpCode.change)
-            myDeserializeChange (currentState, br, out r);
-         else
-         {
-            r = null;
-            // BlueEventLog.logAssert (false, "Illegal operation for this function, can only deal with single row " +
-            //    "add, delete and change");
-         }
-      }
-
-      protected virtual void myDeserializeAdd (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r)
-      {
-         object[] retVal = new object[mySchema.columnInfo.Length];
-         foreach (ColumnInfo ci in mySchema.columnInfo)
-            retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br);
-         r = new BlueRow (mySchema, retVal);
-      }
-
-      protected virtual void myDeserializeRemove (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r)
-      {
-         r = currentState[br.ReadInt32()];
-      }
-
-
-      protected virtual void myDeserializeChange (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r)
-      {
-         MutableBlueRow rv = new MutableBlueRow (currentState[br.ReadInt32 ()]);
-         foreach (int colNo in myChangingColIndices)
-            rv.setValue (colNo, myHelper.deserializeColumn (colNo, br));
-         r = rv;
-      }
-
-   }
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs

@@ -1,90 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-using BCG.Utilities;
-
-namespace BCG
-{
-   /// <summary>
-   /// Specializes in serializing rows that are sent over as a result of a change; only the changed columns are 
-   /// sent together with the id.
-   /// </summary>
-   public class FixedChangingRowSerializer : IBlueRowSerializer
-   {
-      private int[] myChangingColIndices;
-      private PrimitiveSerializationHelper myHelper;
-      private IBlueTableSchema mySchema;
-
-      public FixedChangingRowSerializer (IBlueTableSchema schema, params string[] changingColumns)
-      {
-         myHelper = new PrimitiveSerializationHelper (schema);
-         myChangingColIndices = new int[changingColumns.Length];
-         for (int i = 0; i < changingColumns.Length; i++)
-            myChangingColIndices[i] = schema[changingColumns[i]].colNo;
-         Array.Sort (myChangingColIndices);
-         mySchema = schema;
-      }
-
-      #region IBlueRowSerializer Members
-
-      public void serializeRow (IBlueRow r, byte op, BinaryWriter bw)
-      {
-         if (op == StreamOpCode.add)
-         {
-            mySerializeAdd (r, bw);
-         }
-         else if (op == StreamOpCode.remove)
-         {
-            mySerializeRemove (r, bw);
-         }
-         else if (op == StreamOpCode.change)
-         {
-            mySerializeChange (r, bw);
-         }
-         else
-         {
-            // BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op in this context", op));
-         }
-      }
-
-      #endregion
-
-      /// <summary>
-      /// The whole row is written, one column after the other with the order defined by the schema. There
-      /// are no columns headers
-      /// </summary>
-      /// <param name="r"></param>
-      /// <param name="bw"></param>
-      protected virtual void mySerializeAdd (IBlueRow r, BinaryWriter bw)
-      {
-         foreach (ColumnInfo ci in r.schema.columnInfo)
-            myHelper.serializeColumn (r, ci.colNo, bw);
-      }
-
-      /// <summary>
-      /// Only the id is written
-      /// </summary>
-      /// <param name="r"></param>
-      /// <param name="bw"></param>
-      protected virtual void mySerializeRemove (IBlueRow r, BinaryWriter bw)
-      {
-         myHelper.serializeColumn (r, r.schema["id"].colNo, bw);
-      }
-
-      /// <summary>
-      /// Serializes only the changed columns, alongside the id. So the format is
-      /// {id, changedColumns*}. The order of the changedColumns are sorted according
-      /// to their index in the schema.
-      /// </summary>
-      /// <param name="r"></param>
-      /// <param name="bw"></param>
-      protected virtual void mySerializeChange (IBlueRow r, BinaryWriter bw)
-      {
-         myHelper.serializeColumn (r, r.schema["id"].colNo, bw);
-         foreach (int colNo in myChangingColIndices)
-            myHelper.serializeColumn (r, colNo, bw);
-      }
-
-   }
-
-}

src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs

@@ -1,110 +0,0 @@
-using System;
-using System.Collections;
-using System.IO;
-using BCG.Utilities;
-
-namespace BCG
-{
-   /// <summary>
-   /// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls) 
-   /// and are constantly changing
-   /// </summary>
-   public abstract class FixedChangingSerializer : BlueDataSerializer
-   {
-
-      public FixedChangingSerializer (IBlueTableSchema schema, 
-                                       string[] changingCols,
-                                       string pubsubDeserializer,
-                                       string clientDeserializer)
-         : base (pubsubDeserializer, clientDeserializer)
-      {
-         myRowSerializer = new FixedChangingRowSerializer (schema, changingCols);
-      }
-   }
-
-   #region BidAsk
-
-   public class BidAskSerializer : FixedChangingSerializer
-   {
-      public BidAskSerializer (IBlueTableSchema schema)
-         : base (schema, 
-                  new string[] {"bidAskZone"}, 
-                  "BCG.BidAskPubsubDeserializer",
-                  "BCG.BidAskClientDeserializer"
-         )
-      {}                                              
-   }
-
-   public class BidAskPubsubDeserializer : PubsubFixedChangingDeserializer
-   {
-      public BidAskPubsubDeserializer () : base (new string[] {"bidAskZone"})
-      {}
-   }
-
-   public class BidAskClientDeserializer : ClientFixedChangingDeserializer
-   {
-      public BidAskClientDeserializer () : base (new string[] {"bidAskZone"})
-      {}
-   }
-
-   #endregion
-
-   #region HHMarketPrice
-
-   public class HHMarketPriceSerializer : FixedChangingSerializer
-   {
-      public HHMarketPriceSerializer(IBlueTableSchema schema)
-         : base(schema,
-                  new string[] { "MarketPrice" },
-                  "BCG.HHMarketPricePubsubDeserializer",
-                  "BCG.HHMarketPriceClientDeserializer"
-         )
-      { }
-   }
-
-   public class HHMarketPricePubsubDeserializer : PubsubFixedChangingDeserializer
-   {
-      public HHMarketPricePubsubDeserializer()
-         : base(new string[] { "MarketPrice" })
-      { }
-   }
-
-   public class HHMarketPriceClientDeserializer : ClientFixedChangingDeserializer
-   {
-      public HHMarketPriceClientDeserializer()
-         : base(new string[] { "MarketPrice" })
-      { }
-   }
-
-   #endregion
-
-   #region ThrottledMarketPrice
-
-   public class ThrottledPriceSerializer : FixedChangingSerializer
-   {
-      public ThrottledPriceSerializer(IBlueTableSchema schema)
-         : base(schema,
-                  new string[] { "PRICE" },
-                  "BCG.ThrottledPricePubsubDeserializer",
-                  "BCG.ThrottledPriceClientDeserializer"
-         )
-      { }
-   }
-
-   public class ThrottledPricePubsubDeserializer : PubsubFixedChangingDeserializer
-   {
-      public ThrottledPricePubsubDeserializer()
-         : base(new string[] { "PRICE" })
-      { }
-   }
-
-   public class ThrottledPriceClientDeserializer : ClientFixedChangingDeserializer
-   {
-      public ThrottledPriceClientDeserializer()
-         : base(new string[] { "PRICE" })
-      { }
-   }
-
-   #endregion
-
-}